使用 Superlinked 和 Redis 构建 GenAI 应用

%pip install superlinked

# we are going to create 2 representations of the data
## 1. separate text and ranking for multimodal superlinked embeddings
## 2. full_review_as_text for LLM embedding of stringified review and rating
@schema

class Review:
    id: IdField
    review_text: String
    rating: Integer    
    full_review_as_text: String

# Embed review data separately
review_text_space = TextSimilaritySpace(
    text=review.review_text, model="all-MiniLM-L6-v2")

rating_maximizer_space = NumberSpace(review.rating, min_value=1, 
    max_value=5, mode=Mode.MAXIMUM)

## Embed the full review as text
full_review_as_text_space = TextSimilaritySpace(
    text=review.full_review_as_text, model="all-MiniLM-L6-v2"

# Combine spaces as vector parts to an index.
## Create one for the stringified review 
naive_index = Index([full_review_as_text_space])

## and one for the structured multimodal embeddings
advanced_index = Index([review_text_space, rating_maximizer_space])

openai_config = OpenAIClientConfig(api_key=userdata.get("openai_api_key"),     model="gpt-4o")

# Define your query using dynamic parameters for query text and weights.
## first a query on the naive index - using natural language
naive_query = (
    Query(
        naive_index,
        weights={
            full_review_as_text_space: Param('full_review_as_text_weight')
        },
    )
    .find(review)
    .similar(full_review_as_text_space.text, Param("query_text"))
    .limit(Param('limit'))
    .with_natural_query(Param("natural_query"), openai_config)
)
## and another on the advanced multimodal index - also using natural language
superlinked_query = (
    Query(
        advanced_index,
        weights={
            review_text_space: Param('review_text_weight'),
            rating_maximizer_space: Param('rating_maximizer_weight'),
        },
    )
    .find(review)
    .similar(review_text_space.text, Param("query_text"))
    .limit(Param('limit'))
    .with_natural_query(Param("natural_query"), openai_config)
)

# Run the app
source: InMemorySource = InMemorySource(review, parser=DataFrameParser(schema=review))
executor = InMemoryExecutor(sources=[source], indices=[naive_index, advanced_index]])
app = executor.run()

# Download dataset
data = pd.read_json("https://storage.googleapis.com/superlinked-preview-test-data/amazon_dataset_1000.jsonl",lines=True)

# Ingest data to the framework.
source.put([data])

# query that is based on the LLM embedded reviews
naive_positive_results = app.query(
    naive_query,
    natural_query='High rated quality products',
    limit=10)
naive_positive_results.to_pandas()


# query based on multimodal Superlinked embeddings
superlinked_positive_results = app.query(
    superlinked_query,
    natural_query='High rated quality products',
    limit=10)
superlinked_positive_results.to_pandas()

# Clone the repository
git clone https://github.com/superlinked/superlinked

cd <repo-directory>/server
./tools/init-venv.sh
cd runner
source "$(poetry env info --path)/bin/activate"
cd ..

# Make sure you have your docker engine running and activate the virtual environment
./tools/deploy.py up

from superlinked.framework.dsl.storage.redis_vector_database import RedisVectorDatabase

vector_database = RedisVectorDatabase(
# Your Redis URL without port or any extra fields 
    "<your_redis_host>",
# Port number, which should be an integer
    12315,     
# The params can be found here: https://redis.readthedocs.io/en/stable/connections.html 
    username="<username>",
    password="<password>"
)

# Copy your configuration to app.py
# ...

# Create a data source to bulk load your production data.
config = DataLoaderConfig("https://storage.googleapis.com/superlinked-sample-datasets/amazon_dataset_ext_1000.jsonl", DataFormat.JSON, pandas_read_kwargs={"lines": True, "chunksize": 100})
source = DataLoaderSource(review, config)

executor = RestExecutor(
    # Add your data source 
    sources=[source],
    # Add the indices that contains your configuration 
    indices=[index],
    # Create a REST endpoint for your query.
    queries=[RestQuery(RestDescriptor("naive_query"), naive_query),      
            RestQuery(RestDescriptor("superlinked_query"), superlinked_query)],

    # Connect to Redis 
    vector_database=RedisVectorDatabase()
)

SuperlinkedRegistry.register(executor)

# Trigger the data load 
curl -X POST 'http://localhost:8080/data-loader/review/run'

# Check the status of the loader 
curl -X GET 'http://localhost:8080/data-loader/review/status'

# Send your first query
curl -X POST \
    'http://localhost:8080/api/v1/search/superlinked_query' \
    --header 'Accept: */*' \
    --header 'Content-Type: application/json' \
    --data-raw '{
        "natural_query": "High rated quality products",


        "limit": 10
    }'