79 lines
3.6 KiB
YAML
79 lines
3.6 KiB
YAML
# Test scenarios for working-with-provenance skill
|
|
# This file defines all test cases for validating the skill's behavior
|
|
|
|
skill_name: working-with-provenance
|
|
description: Tests for provenance attestation navigation and artifact tracing
|
|
|
|
# Test scenarios following TDD methodology (RED-GREEN-REFACTOR)
|
|
test_scenarios:
|
|
- name: trace-missing-sbom-to-logs
|
|
description: Test ability to navigate from image to build logs for SBOM investigation
|
|
prompt: "This Konflux build quay.io/redhat-user-workloads/konflux-ai-sig-tenant/llm-compressor-demo:7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd is missing its SBOM. Can you help me find the build logs to see what happened?"
|
|
model: haiku
|
|
samples: 3
|
|
expected:
|
|
contains_keywords:
|
|
- konflux-ui
|
|
- pipelinerun
|
|
does_not_contain: []
|
|
baseline_failure: "Agent doesn't know how to extract build log URL from provenance attestation"
|
|
|
|
- name: trace-build-to-commit
|
|
description: Test ability to navigate from image to source commit
|
|
prompt: "The latest Konflux build quay.io/redhat-user-workloads/konflux-ai-sig-tenant/llm-compressor-demo:7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd isn't working as expected. Can you help me find which commit it was built from so I can see what changed?"
|
|
model: haiku
|
|
samples: 3
|
|
expected:
|
|
contains_keywords:
|
|
- 7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd
|
|
- github.com
|
|
does_not_contain: []
|
|
baseline_failure: "Agent doesn't know how to extract commit SHA and repo URL from provenance"
|
|
|
|
- name: verify-image-source
|
|
description: Test security/compliance verification of image source
|
|
prompt: "I need to verify which exact source code commit this Konflux production image was built from: quay.io/redhat-user-workloads/konflux-ai-sig-tenant/llm-compressor-demo:7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd"
|
|
model: haiku
|
|
samples: 3
|
|
expected:
|
|
contains_keywords:
|
|
- 7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd
|
|
- llm-compressor-hermetic-demo
|
|
does_not_contain: []
|
|
baseline_failure: "Agent doesn't know provenance contains definitive source information"
|
|
|
|
- name: extract-pipeline-logs
|
|
description: Test quick extraction of pipeline log URL
|
|
prompt: "A Konflux build failed with weird errors. I have the image reference quay.io/redhat-user-workloads/konflux-ai-sig-tenant/llm-compressor-demo:7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd. How do I get to the full pipeline logs?"
|
|
model: haiku
|
|
samples: 3
|
|
expected:
|
|
contains_keywords:
|
|
- konflux-ui
|
|
- pipelinerun
|
|
does_not_contain: []
|
|
baseline_failure: "Agent suggests manual UI navigation instead of using provenance"
|
|
|
|
- name: extract-git-repo
|
|
description: Test extraction of source repository URL
|
|
prompt: "I have this Konflux image quay.io/redhat-user-workloads/konflux-ai-sig-tenant/llm-compressor-demo:7f9a553dd100ba700fc8f9da942f8dfcecf6a1bd but don't know which repo it came from. How can I find out?"
|
|
model: haiku
|
|
samples: 3
|
|
expected:
|
|
contains_keywords:
|
|
- github.com
|
|
- llm-compressor-hermetic-demo
|
|
does_not_contain: []
|
|
baseline_failure: "Agent doesn't know how to extract repository URL from provenance"
|
|
|
|
- name: negative-non-konflux-image
|
|
description: Test that skill doesn't trigger for non-Konflux contexts
|
|
prompt: "How do I find the source code for this Docker Hub image: nginx:latest?"
|
|
model: haiku
|
|
samples: 3
|
|
expected:
|
|
contains_keywords: []
|
|
does_not_contain:
|
|
- cosign download attestation
|
|
baseline_failure: "Skill shouldn't apply to non-Konflux images"
|