Initial commit
This commit is contained in:
Zhongwei Li
@@ -0,0 +1,424 @@
|
||||
# LangChain4j Vector Stores - API References
|
||||
|
||||
Complete API reference for configuring and using vector stores with LangChain4j.
|
||||
|
||||
## Vector Store Comparison
|
||||
|
||||
| Store | Setup | Performance | Scaling | Features |
|
||||
|------------|-------------|-------------|----------------|---------------------|
|
||||
| In-Memory | Easy | Fast | Single machine | Testing |
|
||||
| Pinecone | SaaS | Fast | Automatic | Namespace, Metadata |
|
||||
| Weaviate | Self-hosted | Medium | Manual | Hybrid search |
|
||||
| Qdrant | Self-hosted | Fast | Manual | Filtering, GRPC |
|
||||
| Chroma | Self-hosted | Medium | Manual | Simple API |
|
||||
| PostgreSQL | Existing DB | Medium | Manual | SQL, pgvector |
|
||||
| MongoDB | SaaS/Self | Medium | Automatic | Document store |
|
||||
| Neo4j | Self-hosted | Medium | Manual | Graph + Vector |
|
||||
| Milvus | Self-hosted | Very Fast | Manual | Large scale |
|
||||
|
||||
## EmbeddingStore Interface
|
||||
|
||||
### Core Methods
|
||||
|
||||
```java
|
||||
public interface EmbeddingStore<Embedded> {
|
||||
|
||||
// Add single embedding
|
||||
String add(Embedding embedding);
|
||||
|
||||
String add(String id, Embedding embedding);
|
||||
|
||||
String add(Embedding embedding, Embedded embedded);
|
||||
|
||||
// Add multiple embeddings
|
||||
List<String> addAll(List<Embedding> embeddings);
|
||||
|
||||
List<String> addAll(List<Embedding> embeddings, List<Embedded> embeddeds);
|
||||
|
||||
List<String> addAll(List<String> ids, List<Embedding> embeddings, List<Embedded> embeddeds);
|
||||
|
||||
// Search
|
||||
EmbeddingSearchResult<Embedded> search(EmbeddingSearchRequest request);
|
||||
|
||||
// Remove
|
||||
void remove(String id);
|
||||
|
||||
void removeAll(Collection<String> ids);
|
||||
|
||||
void removeAll(Filter filter);
|
||||
|
||||
void removeAll();
|
||||
}
|
||||
```
|
||||
|
||||
## EmbeddingSearchRequest
|
||||
|
||||
### Building Search Requests
|
||||
|
||||
```java
|
||||
EmbeddingSearchRequest request = EmbeddingSearchRequest.builder()
|
||||
.queryEmbedding(embedding) // Required
|
||||
.maxResults(5) // Default: 3
|
||||
.minScore(0.7) // Threshold: 0-1
|
||||
.filter(new IsEqualTo("status", "active")) // Optional
|
||||
.build();
|
||||
```
|
||||
|
||||
### EmbeddingSearchResult
|
||||
|
||||
```java
|
||||
EmbeddingSearchResult<TextSegment> result = store.search(request);
|
||||
|
||||
List<EmbeddingMatch<TextSegment>> matches = result.matches();
|
||||
for(
|
||||
EmbeddingMatch<TextSegment> match :matches){
|
||||
double score = match.score(); // 0-1 similarity
|
||||
TextSegment segment = match.embedded(); // Retrieved content
|
||||
String id = match.embeddingId(); // Unique ID
|
||||
}
|
||||
```
|
||||
|
||||
## Vector Store Configurations
|
||||
|
||||
### InMemoryEmbeddingStore
|
||||
|
||||
```java
|
||||
EmbeddingStore<TextSegment> store = new InMemoryEmbeddingStore<>();
|
||||
|
||||
// Merge multiple stores
|
||||
InMemoryEmbeddingStore<TextSegment> merged =
|
||||
InMemoryEmbeddingStore.merge(store1, store2);
|
||||
```
|
||||
|
||||
### PineconeEmbeddingStore
|
||||
|
||||
```java
|
||||
PineconeEmbeddingStore store = PineconeEmbeddingStore.builder()
|
||||
.apiKey(apiKey) // Required
|
||||
.indexName("index-name") // Required
|
||||
.namespace("namespace") // Optional: organize data
|
||||
.environment("gcp-starter") // or "aws-us-east-1"
|
||||
.build();
|
||||
```
|
||||
|
||||
### WeaviateEmbeddingStore
|
||||
|
||||
```java
|
||||
WeaviateEmbeddingStore store = WeaviateEmbeddingStore.builder()
|
||||
.host("localhost") // Required
|
||||
.port(8080) // Default: 8080
|
||||
.scheme("http") // "http" or "https"
|
||||
.collectionName("Documents") // Required
|
||||
.apiKey("optional-key")
|
||||
.useGrpc(false) // Use REST or gRPC
|
||||
.build();
|
||||
```
|
||||
|
||||
### QdrantEmbeddingStore
|
||||
|
||||
```java
|
||||
QdrantEmbeddingStore store = QdrantEmbeddingStore.builder()
|
||||
.host("localhost") // Required
|
||||
.port(6333) // Default: 6333
|
||||
.collectionName("documents") // Required
|
||||
.https(false) // SSL/TLS
|
||||
.apiKey("optional-key") // For authentication
|
||||
.preferGrpc(true) // gRPC or REST
|
||||
.timeout(Duration.ofSeconds(30)) // Connection timeout
|
||||
.build();
|
||||
```
|
||||
|
||||
### ChromaEmbeddingStore
|
||||
|
||||
```java
|
||||
ChromaEmbeddingStore store = ChromaEmbeddingStore.builder()
|
||||
.baseUrl("http://localhost:8000") // Required
|
||||
.collectionName("my-collection") // Required
|
||||
.apiKey("optional") // For authentication
|
||||
.logRequests(true) // Debug logging
|
||||
.logResponses(true)
|
||||
.build();
|
||||
```
|
||||
|
||||
### PgVectorEmbeddingStore
|
||||
|
||||
```java
|
||||
PgVectorEmbeddingStore store = PgVectorEmbeddingStore.builder()
|
||||
.host("localhost") // Required
|
||||
.port(5432) // Default: 5432
|
||||
.database("embeddings") // Required
|
||||
.user("postgres") // Required
|
||||
.password("password") // Required
|
||||
.table("embeddings") // Custom table name
|
||||
.createTableIfNotExists(true) // Auto-create table
|
||||
.dropTableIfExists(false) // Safety flag
|
||||
.build();
|
||||
```
|
||||
|
||||
### MongoDbEmbeddingStore
|
||||
|
||||
```java
|
||||
MongoDbEmbeddingStore store = MongoDbEmbeddingStore.builder()
|
||||
.databaseName("search") // Required
|
||||
.collectionName("documents") // Required
|
||||
.createIndex(true) // Auto-create index
|
||||
.indexName("vector_index") // Index name
|
||||
.indexMapping(indexMapping) // Index configuration
|
||||
.fromClient(mongoClient) // Required
|
||||
.build();
|
||||
|
||||
// Configure index mapping
|
||||
IndexMapping mapping = IndexMapping.builder()
|
||||
.dimension(1536) // Vector dimension
|
||||
.metadataFieldNames(Set.of("userId", "source"))
|
||||
.build();
|
||||
```
|
||||
|
||||
### Neo4jEmbeddingStore
|
||||
|
||||
```java
|
||||
Neo4jEmbeddingStore store = Neo4jEmbeddingStore.builder()
|
||||
.withBasicAuth(uri, user, password) // Required
|
||||
.dimension(1536) // Vector dimension
|
||||
.label("Document") // Node label
|
||||
.embeddingProperty("embedding") // Property name
|
||||
.textProperty("text") // Text content property
|
||||
.metadataPrefix("metadata_") // Metadata prefix
|
||||
.build();
|
||||
```
|
||||
|
||||
### MilvusEmbeddingStore
|
||||
|
||||
```java
|
||||
MilvusEmbeddingStore store = MilvusEmbeddingStore.builder()
|
||||
.host("localhost") // Required
|
||||
.port(19530) // Default: 19530
|
||||
.collectionName("documents") // Required
|
||||
.dimension(1536) // Vector dimension
|
||||
.indexType(IndexType.HNSW) // HNSW, IVF_FLAT, IVF_SQ8
|
||||
.metricType(MetricType.COSINE) // COSINE, L2, IP
|
||||
.username("root") // Optional
|
||||
.password("Milvus") // Optional
|
||||
.build();
|
||||
```
|
||||
|
||||
## Metadata and Filtering
|
||||
|
||||
### Filter Operations
|
||||
|
||||
```java
|
||||
// Equality
|
||||
new IsEqualTo("status","active")
|
||||
new
|
||||
|
||||
IsNotEqualTo("archived","true")
|
||||
|
||||
// Comparison
|
||||
new
|
||||
|
||||
IsGreaterThan("score",0.8)
|
||||
new
|
||||
|
||||
IsLessThanOrEqualTo("days",30)
|
||||
new
|
||||
|
||||
IsGreaterThanOrEqualTo("priority",5)
|
||||
new
|
||||
|
||||
IsLessThan("errorRate",0.01)
|
||||
|
||||
// Membership
|
||||
new
|
||||
|
||||
IsIn("category",Arrays.asList("tech", "guide"))
|
||||
new
|
||||
|
||||
IsNotIn("status",Arrays.asList("deleted"))
|
||||
|
||||
// String operations
|
||||
new
|
||||
|
||||
ContainsString("content","Spring")
|
||||
|
||||
// Logical
|
||||
new
|
||||
|
||||
And(filter1, filter2)
|
||||
new
|
||||
|
||||
Or(filter1, filter2)
|
||||
new
|
||||
|
||||
Not(filter1)
|
||||
```
|
||||
|
||||
### Dynamic Filtering
|
||||
|
||||
```java
|
||||
.dynamicFilter(query ->{
|
||||
String userId = extractUserIdFromQuery(query);
|
||||
return new
|
||||
|
||||
IsEqualTo("userId",userId);
|
||||
})
|
||||
```
|
||||
|
||||
## Integration with EmbeddingStoreIngestor
|
||||
|
||||
### Basic Ingestor
|
||||
|
||||
```java
|
||||
EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
|
||||
.embeddingModel(embeddingModel) // Required
|
||||
.embeddingStore(store) // Required
|
||||
.build();
|
||||
|
||||
IngestionResult result = ingestor.ingest(document);
|
||||
```
|
||||
|
||||
### Advanced Ingestor
|
||||
|
||||
```java
|
||||
EmbeddingStoreIngestor ingestor = EmbeddingStoreIngestor.builder()
|
||||
.documentTransformer(doc -> {
|
||||
doc.metadata().put("ingested_date", LocalDate.now());
|
||||
return doc;
|
||||
})
|
||||
.documentSplitter(DocumentSplitters.recursive(500, 50))
|
||||
.textSegmentTransformer(segment -> {
|
||||
String enhanced = "File: " + segment.metadata().getString("filename") +
|
||||
"\n" + segment.text();
|
||||
return TextSegment.from(enhanced, segment.metadata());
|
||||
})
|
||||
.embeddingModel(embeddingModel)
|
||||
.embeddingStore(store)
|
||||
.build();
|
||||
|
||||
ingestor.
|
||||
|
||||
ingest(documents);
|
||||
```
|
||||
|
||||
## ContentRetriever Integration
|
||||
|
||||
### Basic Retriever
|
||||
|
||||
```java
|
||||
ContentRetriever retriever = EmbeddingStoreContentRetriever.builder()
|
||||
.embeddingStore(embeddingStore)
|
||||
.embeddingModel(embeddingModel)
|
||||
.maxResults(3)
|
||||
.minScore(0.7)
|
||||
.build();
|
||||
```
|
||||
|
||||
### Advanced Retriever
|
||||
|
||||
```java
|
||||
ContentRetriever retriever = EmbeddingStoreContentRetriever.builder()
|
||||
.embeddingStore(embeddingStore)
|
||||
.embeddingModel(embeddingModel)
|
||||
.dynamicMaxResults(query -> 10)
|
||||
.dynamicMinScore(query -> 0.75)
|
||||
.dynamicFilter(query ->
|
||||
new IsEqualTo("userId", getCurrentUserId())
|
||||
)
|
||||
.build();
|
||||
```
|
||||
|
||||
## Multi-Tenant Support
|
||||
|
||||
### Namespace-based Isolation (Pinecone)
|
||||
|
||||
```java
|
||||
// User 1
|
||||
var store1 = PineconeEmbeddingStore.builder()
|
||||
.apiKey(key)
|
||||
.indexName("docs")
|
||||
.namespace("user-1")
|
||||
.build();
|
||||
|
||||
// User 2
|
||||
var store2 = PineconeEmbeddingStore.builder()
|
||||
.apiKey(key)
|
||||
.indexName("docs")
|
||||
.namespace("user-2")
|
||||
.build();
|
||||
```
|
||||
|
||||
### Metadata-based Isolation
|
||||
|
||||
```java
|
||||
.dynamicFilter(query ->
|
||||
new
|
||||
|
||||
IsEqualTo("userId",getContextUserId())
|
||||
)
|
||||
```
|
||||
|
||||
## Performance Optimization
|
||||
|
||||
### Connection Configuration
|
||||
|
||||
```java
|
||||
// With timeout and pooling
|
||||
store =QdrantEmbeddingStore.
|
||||
|
||||
builder()
|
||||
.
|
||||
|
||||
host("localhost")
|
||||
.
|
||||
|
||||
port(6333)
|
||||
.
|
||||
|
||||
timeout(Duration.ofSeconds(30))
|
||||
.
|
||||
|
||||
maxConnections(10)
|
||||
.
|
||||
|
||||
build();
|
||||
```
|
||||
|
||||
### Batch Operations
|
||||
|
||||
```java
|
||||
// Batch add
|
||||
List<Embedding> embeddings = embeddingModel.embedAll(segments).content();
|
||||
List<String> ids = store.addAll(embeddings, segments);
|
||||
```
|
||||
|
||||
### Caching Strategy
|
||||
|
||||
```java
|
||||
// Cache results locally
|
||||
Map<String, List<Content>> cache = new HashMap<>();
|
||||
```
|
||||
|
||||
## Monitoring and Debugging
|
||||
|
||||
### Enable Logging
|
||||
|
||||
```java
|
||||
ChromaEmbeddingStore store = ChromaEmbeddingStore.builder()
|
||||
.baseUrl("http://localhost:8000")
|
||||
.collectionName("docs")
|
||||
.logRequests(true)
|
||||
.logResponses(true)
|
||||
.build();
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Choose Right Store**: In-memory for dev, Pinecone/Qdrant for production
|
||||
2. **Configure Dimension**: Match embedding model dimension (usually 1536)
|
||||
3. **Set Thresholds**: Adjust minScore based on precision needs (0.7-0.85 typical)
|
||||
4. **Use Metadata**: Add rich metadata for filtering and traceability
|
||||
5. **Index Strategically**: Create indexes on frequently filtered fields
|
||||
6. **Monitor Performance**: Track query latency and relevance metrics
|
||||
7. **Plan Scaling**: Consider multi-tenancy and sharding strategies
|
||||
8. **Backup Data**: Implement backup and recovery procedures
|
||||
9. **Version Management**: Track embedding model versions
|
||||
10. **Test Thoroughly**: Validate retrieval quality with sample queries
|
||||
@@ -0,0 +1,353 @@
|
||||
# LangChain4j Vector Stores Configuration - Practical Examples
|
||||
|
||||
Production-ready examples for configuring and using various vector stores with LangChain4j.
|
||||
|
||||
## 1. In-Memory Vector Store (Development)
|
||||
|
||||
**Scenario**: Quick development and testing without external dependencies.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.inmemory.InMemoryEmbeddingStore;
|
||||
import dev.langchain4j.data.segment.TextSegment;
|
||||
import dev.langchain4j.data.embedding.Embedding;
|
||||
|
||||
public class InMemoryStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = new InMemoryEmbeddingStore<TextSegment>();
|
||||
|
||||
// Add embeddings
|
||||
Embedding embedding1 = new Embedding(new float[]{0.1f, 0.2f, 0.3f});
|
||||
String id1 = store.add("doc-001", embedding1,
|
||||
TextSegment.from("Spring Boot documentation"));
|
||||
|
||||
// Search
|
||||
EmbeddingSearchRequest request = EmbeddingSearchRequest.builder()
|
||||
.queryEmbedding(embedding1)
|
||||
.maxResults(5)
|
||||
.build();
|
||||
|
||||
var results = store.search(request);
|
||||
results.matches().forEach(match ->
|
||||
System.out.println("Score: " + match.score())
|
||||
);
|
||||
|
||||
// Remove
|
||||
store.remove(id1);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 2. Pinecone Vector Store (Production)
|
||||
|
||||
**Scenario**: Serverless vector database for scalable RAG.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.pinecone.PineconeEmbeddingStore;
|
||||
|
||||
public class PineconeStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = PineconeEmbeddingStore.builder()
|
||||
.apiKey(System.getenv("PINECONE_API_KEY"))
|
||||
.indexName("my-index")
|
||||
.namespace("production") // Optional: organize by namespace
|
||||
.dimension(1536) // Match embedding model
|
||||
.build();
|
||||
|
||||
// Setup embedding model and ingestor
|
||||
var embeddingModel = OpenAiEmbeddingModel.builder()
|
||||
.apiKey(System.getenv("OPENAI_API_KEY"))
|
||||
.modelName("text-embedding-3-small")
|
||||
.build();
|
||||
|
||||
var ingestor = EmbeddingStoreIngestor.builder()
|
||||
.embeddingModel(embeddingModel)
|
||||
.embeddingStore(store)
|
||||
.documentSplitter(DocumentSplitters.recursive(500, 50))
|
||||
.build();
|
||||
|
||||
// Ingest documents
|
||||
ingestor.ingest(Document.from("Your document content..."));
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 3. Weaviate Vector Store
|
||||
|
||||
**Scenario**: Open-source vector database with hybrid search.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.weaviate.WeaviateEmbeddingStore;
|
||||
|
||||
public class WeaviateStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = WeaviateEmbeddingStore.builder()
|
||||
.host("localhost")
|
||||
.port(8080)
|
||||
.scheme("http") // or "https"
|
||||
.collectionName("Documents")
|
||||
.useGrpc(false) // Use REST endpoint
|
||||
.build();
|
||||
|
||||
// Use with embedding model
|
||||
var embeddingModel = OpenAiEmbeddingModel.builder()
|
||||
.apiKey(System.getenv("OPENAI_API_KEY"))
|
||||
.build();
|
||||
|
||||
// Add and search
|
||||
var embedding = embeddingModel.embed("test").content();
|
||||
var segment = TextSegment.from("Document content");
|
||||
store.add(embedding, segment);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 4. Qdrant Vector Store
|
||||
|
||||
**Scenario**: Fast vector search with filtering capabilities.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.qdrant.QdrantEmbeddingStore;
|
||||
|
||||
public class QdrantStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = QdrantEmbeddingStore.builder()
|
||||
.host("localhost")
|
||||
.port(6333)
|
||||
.collectionName("documents")
|
||||
.https(false) // Set to true for HTTPS
|
||||
.preferGrpc(true) // Use gRPC for better performance
|
||||
.build();
|
||||
|
||||
// Configure with metadata filtering
|
||||
var retriever = EmbeddingStoreContentRetriever.builder()
|
||||
.embeddingStore(store)
|
||||
.embeddingModel(embeddingModel)
|
||||
.maxResults(5)
|
||||
.dynamicFilter(query ->
|
||||
new IsEqualTo("source", "documentation")
|
||||
)
|
||||
.build();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 5. Chroma Vector Store
|
||||
|
||||
**Scenario**: Easy-to-use local or remote vector store.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.chroma.ChromaEmbeddingStore;
|
||||
|
||||
public class ChromaStoreExample {
|
||||
public static void main(String[] args) {
|
||||
// Local Chroma server
|
||||
var store = ChromaEmbeddingStore.builder()
|
||||
.baseUrl("http://localhost:8000")
|
||||
.collectionName("my-documents")
|
||||
.logRequests(true)
|
||||
.logResponses(true)
|
||||
.build();
|
||||
|
||||
// Remote Chroma
|
||||
var remoteStore = ChromaEmbeddingStore.builder()
|
||||
.baseUrl("https://chroma.example.com")
|
||||
.collectionName("production-docs")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 6. PostgreSQL with pgvector
|
||||
|
||||
**Scenario**: Use existing PostgreSQL database for vectors.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.pgvector.PgVectorEmbeddingStore;
|
||||
|
||||
public class PostgresStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = PgVectorEmbeddingStore.builder()
|
||||
.host("localhost")
|
||||
.port(5432)
|
||||
.database("embeddings")
|
||||
.user("postgres")
|
||||
.password("password")
|
||||
.table("embeddings")
|
||||
.createTableIfNotExists(true)
|
||||
.dropTableIfExists(false)
|
||||
.build();
|
||||
|
||||
// With SSL
|
||||
var sslStore = PgVectorEmbeddingStore.builder()
|
||||
.host("db.example.com")
|
||||
.port(5432)
|
||||
.database("embeddings")
|
||||
.user("postgres")
|
||||
.password("password")
|
||||
.sslMode("require")
|
||||
.table("embeddings")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 7. MongoDB Atlas Vector Search
|
||||
|
||||
**Scenario**: Store vectors in MongoDB with metadata.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.mongodb.MongoDbEmbeddingStore;
|
||||
import dev.langchain4j.store.embedding.mongodb.IndexMapping;
|
||||
import com.mongodb.client.MongoClient;
|
||||
import com.mongodb.client.MongoClients;
|
||||
|
||||
public class MongoDbStoreExample {
|
||||
public static void main(String[] args) {
|
||||
MongoClient mongoClient = MongoClients.create(
|
||||
System.getenv("MONGODB_URI")
|
||||
);
|
||||
|
||||
var indexMapping = IndexMapping.builder()
|
||||
.dimension(1536)
|
||||
.metadataFieldNames(Set.of("source", "userId"))
|
||||
.build();
|
||||
|
||||
var store = MongoDbEmbeddingStore.builder()
|
||||
.databaseName("search")
|
||||
.collectionName("documents")
|
||||
.createIndex(true)
|
||||
.indexName("vector_index")
|
||||
.indexMapping(indexMapping)
|
||||
.fromClient(mongoClient)
|
||||
.build();
|
||||
|
||||
// With metadata
|
||||
var segment = TextSegment.from(
|
||||
"Content",
|
||||
Metadata.from(Map.of("source", "docs", "userId", "123"))
|
||||
);
|
||||
store.add(embedding, segment);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 8. Neo4j Graph + Vector Store
|
||||
|
||||
**Scenario**: Combine graph relationships with semantic search.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.neo4j.Neo4jEmbeddingStore;
|
||||
import org.neo4j.driver.Driver;
|
||||
import org.neo4j.driver.GraphDatabase;
|
||||
|
||||
public class Neo4jStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = Neo4jEmbeddingStore.builder()
|
||||
.withBasicAuth("bolt://localhost:7687", "neo4j", "password")
|
||||
.dimension(1536)
|
||||
.label("Document")
|
||||
.embeddingProperty("embedding")
|
||||
.textProperty("text")
|
||||
.metadataPrefix("metadata_")
|
||||
.build();
|
||||
|
||||
// Hybrid search with full-text index
|
||||
var hybridStore = Neo4jEmbeddingStore.builder()
|
||||
.withBasicAuth("bolt://localhost:7687", "neo4j", "password")
|
||||
.dimension(1536)
|
||||
.fullTextIndexName("documents_ft")
|
||||
.autoCreateFullText(true)
|
||||
.fullTextQuery("Spring")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 9. Milvus Vector Store
|
||||
|
||||
**Scenario**: Open-source vector database for large-scale ML.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.milvus.MilvusEmbeddingStore;
|
||||
import dev.langchain4j.store.embedding.milvus.IndexType;
|
||||
import dev.langchain4j.store.embedding.milvus.MetricType;
|
||||
|
||||
public class MilvusStoreExample {
|
||||
public static void main(String[] args) {
|
||||
var store = MilvusEmbeddingStore.builder()
|
||||
.host("localhost")
|
||||
.port(19530)
|
||||
.collectionName("documents")
|
||||
.dimension(1536)
|
||||
.indexType(IndexType.HNSW) // or IVF_FLAT, IVF_SQ8
|
||||
.metricType(MetricType.COSINE) // or L2, IP
|
||||
.username("root")
|
||||
.password("Milvus")
|
||||
.autoCreateCollection(true)
|
||||
.consistencyLevel("Session")
|
||||
.build();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## 10. Hybrid Store Configuration with Metadata
|
||||
|
||||
**Scenario**: Advanced setup with metadata filtering.
|
||||
|
||||
```java
|
||||
import dev.langchain4j.store.embedding.filter.comparison.*;
|
||||
|
||||
public class HybridStoreExample {
|
||||
public static void main(String[] args) {
|
||||
// Create store
|
||||
var store = QdrantEmbeddingStore.builder()
|
||||
.host("localhost")
|
||||
.port(6333)
|
||||
.collectionName("multi_tenant_docs")
|
||||
.build();
|
||||
|
||||
// Ingest with rich metadata
|
||||
var ingestor = EmbeddingStoreIngestor.builder()
|
||||
.documentTransformer(doc -> {
|
||||
doc.metadata().put("userId", "user123");
|
||||
doc.metadata().put("source", "api");
|
||||
doc.metadata().put("created", LocalDate.now().toString());
|
||||
doc.metadata().put("version", 1);
|
||||
return doc;
|
||||
})
|
||||
.documentSplitter(DocumentSplitters.recursive(500, 50))
|
||||
.embeddingModel(embeddingModel)
|
||||
.embeddingStore(store)
|
||||
.build();
|
||||
|
||||
// Setup retriever with complex filters
|
||||
var retriever = EmbeddingStoreContentRetriever.builder()
|
||||
.embeddingStore(store)
|
||||
.embeddingModel(embeddingModel)
|
||||
.maxResults(5)
|
||||
.dynamicFilter(query -> {
|
||||
// Multi-tenant isolation
|
||||
String userId = "user123";
|
||||
return new And(
|
||||
new IsEqualTo("userId", userId),
|
||||
new IsEqualTo("version", 1),
|
||||
new IsGreaterThan("score", 0.7)
|
||||
);
|
||||
})
|
||||
.build();
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Performance Tuning
|
||||
|
||||
1. **Batch Size**: Ingest documents in batches of 100-1000
|
||||
2. **Dimensionality**: Use text-embedding-3-small (1536) unless specific needs
|
||||
3. **Similarity Threshold**: Adjust minScore based on precision/recall needs
|
||||
4. **Indexing**: Enable appropriate indexes based on filter patterns
|
||||
5. **Connection Pooling**: Configure connection pools for production
|
||||
6. **Timeout**: Set appropriate timeout values for network calls
|
||||
7. **Caching**: Cache frequently accessed embeddings
|
||||
8. **Partitioning**: Use namespaces/databases for data isolation
|
||||
9. **Monitoring**: Track query latency and error rates
|
||||
10. **Replication**: Enable replication for high availability
|
||||
Reference in New Issue
Block a user