Skip to the content.

Configuration Schema Reference (mimic.json)

mimic.json is the configuration file for the OpenMimic generation engine. It is strictly typed and governed by proto/mimic_config.proto.


📄 Complete Example

{
  "id": "gen-signups-prod",
  "name": "User Signups Stream",
  "seed": 42,
  "event_limit": 1000,
  "rate_limit_per_second": 25,
  "sink": {
    "kafka": {
      "brokers": ["localhost:9092"],
      "topic": "user.signups",
      "client_id": "openmimic-engine"
    }
  },
  "conventions": {
    "tenant_id": "tenant-corp-a",
    "topic_namespace": "prod.us-east",
    "inject_audit_columns": true
  },
  "telemetry": {
    "enable_w3c_traceparent": true,
    "metrics_port": 8080
  },
  "dlq": {
    "sink_type": "DLQ_SINK_BUFFER",
    "dlq_topic": "dlq.invalid_events",
    "max_buffer_size": 1000
  },
  "fields": [
    {
      "name": "user_id",
      "generator": {
        "uuid": { "uppercase": false }
      }
    },
    {
      "name": "email",
      "generator": {
        "string_faker": { "category": "email" }
      }
    },
    {
      "name": "status",
      "generator": {
        "weighted_choice": {
          "options": [
            { "value": "SUCCESS", "weight": 95.0 },
            { "value": "FAILURE", "weight": 5.0 }
          ]
        }
      }
    },
    {
      "name": "created_at",
      "generator": {
        "now": { "format": "RFC3339" }
      }
    }
  ]
}

⚙️ Top-Level Properties

Property Type Required Description
id string Optional Unique identifier for the generator instance. Auto-generated if omitted.
name string Required Human-readable label displayed in Studio.
seed int64 Optional PRNG seed for deterministic output. Defaults to current epoch nanoseconds.
event_limit int64 Optional Total events to generate. 0 indicates an unbounded, infinite stream.
rate_limit_per_second int32 Optional Max events generated per second. 0 indicates maximum throughput without artificial sleep.
sink SinkConfig Required Destination where events will be delivered.
fields repeated Field Required List of field definitions to synthesize.

🎯 Sink Configuration

The sink object accepts one of the following definitions:

1. Kafka Sink

"sink": {
  "kafka": {
    "brokers": ["localhost:9092"],
    "topic": "events.raw",
    "client_id": "mimic-client",
    "schema_registry": {
      "enabled": true,
      "url": "http://localhost:8081",
      "format": "avro",
      "subject_naming_strategy": "TopicRecordNameStrategy",
      "record_name": "io.confluent.examples.OrderEvent"
    },
    "transactional": {
      "enabled": true,
      "transactional_id": "mimic-tx-producer-01",
      "transaction_timeout_ms": 60000,
      "batch_commit_size": 100
    }
  }
}

2. PostgreSQL Sink

"sink": {
  "postgres": {
    "connection_string": "postgres://user:pass@localhost:5432/db",
    "table": "events",
    "columns": ["id", "email", "status"]
  }
}

Note: If columns is omitted, the payload is written directly to a single data JSONB column.

3. Amazon S3 / Cloudflare R2 / MinIO Sink

"sink": {
  "s3": {
    "bucket": "data-lake-raw",
    "region": "us-east-1",
    "prefix": "incoming/events/",
    "file_format": "parquet",
    "compression_codec": "zstd",
    "max_records_per_file": 10000,
    "flush_interval_seconds": 60
  }
}

4. Snowflake Streaming Sink

"sink": {
  "snowflake": {
    "account_identifier": "xy12345.us-east-1",
    "warehouse": "COMPUTE_WH",
    "database": "ANALYTICS_DB",
    "schema": "PUBLIC",
    "table": "RAW_EVENTS",
    "batch_size": 500,
    "flush_interval_ms": 5000
  }
}

5. Google BigQuery Streaming Sink

"sink": {
  "bigquery": {
    "project_id": "my-gcp-project",
    "dataset_id": "telemetry",
    "table_id": "streaming_events",
    "batch_size": 500,
    "write_mode": "STORAGE_WRITE_API"
  }
}

6. Apache Iceberg Streaming Lakehouse Sink

"sink": {
  "iceberg": {
    "catalog_uri": "http://localhost:8181",
    "warehouse": "s3://mimic-lakehouse",
    "database": "analytics",
    "table": "user_orders",
    "format_version": "2",
    "commit_interval_ms": 5000
  }
}

7. Delta Lake Streaming ACID Sink

"sink": {
  "delta_lake": {
    "table_uri": "s3://lakehouse/delta/events",
    "mode": "append",
    "batch_size": 1000,
    "generate_delta_log": true
  }
}

8. GraphQL & gRPC Sinks

9. Enterprise SSO & Identity Provider (sso_config)

"sso_config": {
  "enabled": true,
  "provider_type": "OKTA",
  "issuer_url": "https://techwasti.okta.com/oauth2/default",
  "client_id": "0oae9k2z45OpenMimicClient",
  "group_role_mappings": {
    "DataPlatform-Admins": "super_admin",
    "Data-Engineers": "tenant_admin",
    "Security-Auditors": "viewer"
  },
  "default_role": "viewer",
  "enforce_pkce": true
}

8. Redis Sink

"sink": {
  "redis": {
    "address": "localhost:6379",
    "key_prefix": "events:order:",
    "key_field": "order_id",
    "ttl_seconds": 3600
  }
}

9. ClickHouse Sink

"sink": {
  "clickhouse": {
    "address": "localhost:9000",
    "database": "default",
    "table": "events_stream",
    "batch_size": 1000
  }
}

10. Stdout Sink

"sink": {
  "stdout": {
    "pretty": false
  }
}

Outputs NDJSON (newline-delimited JSON) to standard output.


⚡ Scale-Out Cluster & CLI Capabilities

1. Distributed Partitioned Generation

Run across $N$ parallel worker agents partitioned deterministically:

mimic-cli cluster --config mimic.json --workers 4 --total 1000000 --rate 5000

2. Reverse-Engineering from Sample DB Records

Profile existing JSON database records to automatically produce generator definitions:

mimic-cli profile --table orders --sample sample_db_rows.json

3. Prometheus & OpenTelemetry Telemetry

Scrape standard Prometheus metrics and propagate W3C distributed trace context:

curl http://localhost:8080/metrics

Headers injected into outgoing messages: