{
"Comment": "Batch ingestion of multiple documents concurrently using a Map state with MaxConcurrency 3. Supports scheduled full-corpus re-ingestion and large document library imports.",
"StartAt": "BulkIngestDocuments",
"States": {
"BulkIngestDocuments": {
"Type": "Map",
"ItemsPath": "$.documents",
"MaxConcurrency": 3,
"ItemProcessor": {
"ProcessorConfig": {
"Mode": "INLINE"
},
"StartAt": "BatchIngestDocument",
"States": {
"BatchIngestDocument": {
"Type": "Task",
"Resource": "${IngestDocumentFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 3,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"Next": "BatchExtractText"
},
"BatchExtractText": {
"Type": "Task",
"Resource": "${ExtractTextContentFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 3,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"Next": "BatchGenerateEmbeddings"
},
"BatchGenerateEmbeddings": {
"Type": "Task",
"Resource": "${GenerateEmbeddingsFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 3,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"Next": "BatchIndexToVectorStore"
},
"BatchIndexToVectorStore": {
"Type": "Task",
"Resource": "${IndexToVectorStoreFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 3,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"Next": "BatchValidateRetrieval"
},
"BatchValidateRetrieval": {
"Type": "Task",
"Resource": "${ValidateRetrievalQualityFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 3,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"Next": "BatchParallelCatalogAndNotify"
},
"BatchParallelCatalogAndNotify": {
"Type": "Parallel",
"Branches": [
{
"StartAt": "BatchUpdateCatalog",
"States": {
"BatchUpdateCatalog": {
"Type": "Task",
"Resource": "${UpdateKnowledgeCatalogFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 3,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"End": true
}
}
},
{
"StartAt": "BatchNotifyStatus",
"States": {
"BatchNotifyStatus": {
"Type": "Task",
"Resource": "${NotifyIngestionStatusFunctionArn}",
"Retry": [
{
"ErrorEquals": [
"States.ALL"
],
"IntervalSeconds": 2,
"MaxAttempts": 2,
"BackoffRate": 2
}
],
"End": true
}
}
}
],
"End": true
}
}
},
"Next": "BulkIngestionComplete"
},
"BulkIngestionComplete": {
"Type": "Succeed"
}
}
}Expand
100%
AI teams can use patterns like this to build reliable, compliant, and scalable automation for payment systems and can test and refine these flows locally with Thrubit to reduce cloud cost and speed up iteration.