services: [] run: - command: ./my-app/run-app.sh test: generation: parquet: - options: path: /tmp/parquet/accounts fields: - name: account_id validation: parquet: - options: path: /tmp/parquet/accounts validations: - expr: ISNOTNULL(account_id) - aggType: count aggExpr: count == 1000

services: - name: postgres #define external services data: my-data/sql #initial service setup (i.e. schema/tables, topics, queues) run: - command: ./my-app/run-postgres-extract-app.sh #how to run your application/job env: #environment variables for your application/job POSTGRES_URL: jdbc:postgresql://postgres:5432/docker test: env: #environment variables for data generation/validation POSTGRES_URL: jdbc:postgresql://postgres:5432/docker mount: #volume mount for data validation - ${PWD}/example/my-app/shared/generated:/opt/app/shared/generated relationship: #generate data with same values used across different data sources postgres_balance.account_number: #ensure account_number in balance table exists when transaction created - postgres_transaction.account_number generation: #define data sources for data generation postgres: - name: postgres_transaction #give it a name to use in relationship definition options: #configuration on specific data source dbtable: account.transactions count: #how many records to generate (1,000 by default) perField: #generate 5 records per account_number fieldNames: [account_number] count: 5 fields: #fields of the data source - name: account_number #default data type is string - name: create_time type: timestamp - name: transaction_id - name: amount type: double - name: postgres_balance options: dbtable: account.balances fields: - name: account_number options: #additional metadata for data generation isUnique: true regex: ACC[0-9]{10} - name: create_time type: timestamp - name: account_status options: oneOf: [open, closed] - name: balance type: double validation: csv: #define data source for data validations - options: path: /opt/app/shared/generated/balances.csv header: true validations: #list of validation to run, can be basic SQL, aggregations, upstream data source or column name validations - expr: ISNOTNULL(account_number) - aggType: count aggExpr: count == 1000 - options: path: /opt/app/shared/generated/transactions.csv header: true validations: - expr: ISNOTNULL(account_number) - aggType: count aggExpr: count == 5000 - groupByCols: [account_number] aggType: count aggExpr: count == 5

name: Integration Test on: push: branches: - * jobs: integration-test: name: Integration Test runs-on: ubuntu-latest steps: - name: Run integration tests uses: data-catering/insta-integration@v1 with: configuration_file: my/custom/folder/insta-integration.yaml insta_infra_folder: insta-infra/folder base_folder: execution/folder data_caterer_version: 0.17.3

- name: Run integration tests id: test-action uses: data-catering/insta-integration@v6 - name: Print Output id: output run: |  echo "Records generated: ${{ steps.test-action.outputs.num_records_generated }}"  echo "Successful validations: ${{ steps.test-action.outputs.num_success_validations }}"  echo "Failed validations: ${{ steps.test-action.outputs.num_failed_validations }}"  echo "Number of validations: ${{ steps.test-action.outputs.num_validations }}"  echo "Validation success rate: ${{ steps.test-action.outputs.validation_success_rate }}"