Initial commit
This commit is contained in:
Zhongwei Li
15
.claude-plugin/plugin.json
Normal file
15
.claude-plugin/plugin.json
Normal file
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"name": "cdp-unification",
|
||||||
|
"description": "Unify and consolidate data across multiple sources",
|
||||||
|
"version": "0.0.0-2025.11.28",
|
||||||
|
"author": {
|
||||||
|
"name": "@cdp-tools-marketplace",
|
||||||
|
"email": "zhongweili@tubi.tv"
|
||||||
|
},
|
||||||
|
"agents": [
|
||||||
|
"./agents"
|
||||||
|
],
|
||||||
|
"commands": [
|
||||||
|
"./commands"
|
||||||
|
]
|
||||||
|
}
|
||||||
3
README.md
Normal file
3
README.md
Normal file
@@ -0,0 +1,3 @@
|
|||||||
|
# cdp-unification
|
||||||
|
|
||||||
|
Unify and consolidate data across multiple sources
|
||||||
472
agents/dynamic-prep-creation.md
Normal file
472
agents/dynamic-prep-creation.md
Normal file
@@ -0,0 +1,472 @@
|
|||||||
|
---
|
||||||
|
name: dynamic-prep-creation
|
||||||
|
description: FOLLOW INSTRUCTIONS EXACTLY - NO THINKING, NO MODIFICATIONS, NO IMPROVEMENTS
|
||||||
|
model: sonnet
|
||||||
|
color: yellow
|
||||||
|
---
|
||||||
|
|
||||||
|
# Dynamic Prep Creation Agent
|
||||||
|
|
||||||
|
## ⚠️ READ THIS FIRST ⚠️
|
||||||
|
**YOUR ONLY JOB: COPY THE EXACT TEMPLATES BELOW**
|
||||||
|
**DO NOT THINK. DO NOT MODIFY. DO NOT IMPROVE.**
|
||||||
|
**JUST COPY THE EXACT TEXT FROM THE TEMPLATES.**
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Copy the exact templates below without any changes.
|
||||||
|
|
||||||
|
**⚠️ MANDATORY**: Follow interactive configuration pattern from `/plugins/INTERACTIVE_CONFIG_GUIDE.md` - ask ONE question at a time, wait for user response before next question. See guide for complete list of required parameters.
|
||||||
|
|
||||||
|
## Critical Files to Create (ALWAYS)
|
||||||
|
|
||||||
|
### 0. Directory Structure (FIRST)
|
||||||
|
**MUST create directories before files**:
|
||||||
|
- Create `unification/config/` directory if it doesn't exist
|
||||||
|
- Create `unification/queries/` directory if it doesn't exist
|
||||||
|
|
||||||
|
### 1. unification/dynmic_prep_creation.dig (Root Directory)
|
||||||
|
**⚠️ FILENAME CRITICAL: MUST be "dynmic_prep_creation.dig" ⚠️**
|
||||||
|
**MUST be created EXACTLY AS IS** - This is production-critical generic code:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
timezone: America/Chicago
|
||||||
|
|
||||||
|
# schedule:
|
||||||
|
# cron>: '0 * * * *'
|
||||||
|
|
||||||
|
_export:
|
||||||
|
!include : config/environment.yml
|
||||||
|
!include : config/src_prep_params.yml
|
||||||
|
td:
|
||||||
|
database: ${client_short_name}_${src}
|
||||||
|
|
||||||
|
+start:
|
||||||
|
_parallel: true
|
||||||
|
+create_schema:
|
||||||
|
td>: queries/create_schema.sql
|
||||||
|
database: ${client_short_name}_${stg}
|
||||||
|
|
||||||
|
+empty_tbl_unif_prep_config:
|
||||||
|
td_ddl>:
|
||||||
|
empty_tables: ["${client_short_name}_${stg}.unif_prep_config"]
|
||||||
|
database: ${client_short_name}_${stg}
|
||||||
|
|
||||||
|
+parse_config:
|
||||||
|
_parallel: true
|
||||||
|
td_for_each>: queries/loop_on_tables.sql
|
||||||
|
_do:
|
||||||
|
|
||||||
|
+store_sqls_in_config:
|
||||||
|
td>:
|
||||||
|
query: select '${td.each.src_db}' as src_db, '${td.each.src_tbl}' as src_tbl,'${td.each.snk_db}' as snk_db,'${td.each.snk_tbl}' as snk_tbl,'${td.each.unif_input_tbl}' as unif_input_tbl,'${td.each.prep_tbl_sql_string}' as prep_tbl_sql_string, '${td.each.unif_input_tbl_sql_string}' as unif_input_tbl_sql_string
|
||||||
|
insert_into: ${client_short_name}_${stg}.unif_prep_config
|
||||||
|
database: ${client_short_name}_${stg}
|
||||||
|
|
||||||
|
+insrt_prep:
|
||||||
|
td>:
|
||||||
|
query: ${td.each.prep_tbl_sql_string.replaceAll("''", "'")}
|
||||||
|
database: ${client_short_name}_${stg}
|
||||||
|
|
||||||
|
+insrt_unif_input_tbl:
|
||||||
|
td>:
|
||||||
|
query: ${td.each.unif_input_tbl_sql_string.replaceAll("''", "'")}
|
||||||
|
database: ${client_short_name}_${stg}
|
||||||
|
|
||||||
|
+unif_input_tbl:
|
||||||
|
td>: queries/unif_input_tbl.sql
|
||||||
|
database: ${client_short_name}_${stg}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 2. unification/queries/create_schema.sql (Queries Directory)
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - NO CHANGES ⚠️**
|
||||||
|
**Generic schema creation - DO NOT MODIFY ANY PART:**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
create table if not exists ${client_short_name}_${stg}.${globals.unif_input_tbl}
|
||||||
|
(
|
||||||
|
source varchar
|
||||||
|
)
|
||||||
|
;
|
||||||
|
|
||||||
|
|
||||||
|
create table if not exists ${client_short_name}_${stg}.${globals.unif_input_tbl}_tmp_td
|
||||||
|
(
|
||||||
|
source varchar
|
||||||
|
)
|
||||||
|
;
|
||||||
|
|
||||||
|
|
||||||
|
create table if not exists ${client_short_name}_${lkup}.exclusion_list
|
||||||
|
(
|
||||||
|
key_name varchar,
|
||||||
|
key_value varchar
|
||||||
|
)
|
||||||
|
;
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. unification/queries/loop_on_tables.sql (Queries Directory)
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - COMPLEX PRODUCTION SQL ⚠️**
|
||||||
|
**Generic loop logic - DO NOT MODIFY A SINGLE CHARACTER:**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
with config as
|
||||||
|
(
|
||||||
|
select cast(
|
||||||
|
json_parse('${prep_tbls}')
|
||||||
|
as array(json)
|
||||||
|
)
|
||||||
|
as tbls_list
|
||||||
|
)
|
||||||
|
, parsed_config as
|
||||||
|
(
|
||||||
|
select *,
|
||||||
|
JSON_EXTRACT_SCALAR(tbl, '$.src_db') as src_db,
|
||||||
|
JSON_EXTRACT_SCALAR(tbl, '$.src_tbl') as src_tbl,
|
||||||
|
JSON_EXTRACT_SCALAR(tbl, '$.snk_db') as snk_db,
|
||||||
|
JSON_EXTRACT_SCALAR(tbl, '$.snk_tbl') as snk_tbl,
|
||||||
|
cast(JSON_EXTRACT(tbl, '$.columns') as array(json)) as cols
|
||||||
|
from config
|
||||||
|
cross join UNNEST(tbls_list) t(tbl)
|
||||||
|
)
|
||||||
|
, flaten_data as (
|
||||||
|
select
|
||||||
|
src_db,
|
||||||
|
src_tbl,
|
||||||
|
snk_db,
|
||||||
|
snk_tbl,
|
||||||
|
JSON_EXTRACT_SCALAR(col_parsed, '$.name') as src_col,
|
||||||
|
JSON_EXTRACT_SCALAR(col_parsed, '$.alias_as') as alias_as
|
||||||
|
from parsed_config
|
||||||
|
cross join UNNEST(cols) t(col_parsed)
|
||||||
|
)
|
||||||
|
, flaten_data_agg as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
src_db, src_tbl, snk_db, snk_tbl,
|
||||||
|
'${globals.unif_input_tbl}_tmp_td' as unif_input_tbl,
|
||||||
|
ARRAY_JOIN(TRANSFORM(ARRAY_AGG(src_col order by src_col), x -> 'cast(' || trim(x) || ' as varchar)'), ', ') as src_cols,
|
||||||
|
ARRAY_JOIN(ARRAY_AGG('cast(' || src_col || ' as varchar) as ' || alias_as order by alias_as), ', ') as col_with_alias,
|
||||||
|
ARRAY_JOIN(ARRAY_AGG(alias_as order by alias_as), ', ') as prep_cols,
|
||||||
|
ARRAY_JOIN(TRANSFORM(ARRAY_AGG(src_col order by src_col), x -> 'coalesce(cast(' || trim(x) || ' as varchar), '''''''')'), '||''''~''''||') as src_key,
|
||||||
|
ARRAY_JOIN(TRANSFORM(ARRAY_AGG(alias_as order by src_col), x -> 'coalesce(cast(' || trim(x) || ' as varchar), '''''''')' ), '||''''~''''||') as prep_key
|
||||||
|
from flaten_data
|
||||||
|
group by src_db, src_tbl, snk_db, snk_tbl
|
||||||
|
)
|
||||||
|
, prep_table_sqls as (
|
||||||
|
select
|
||||||
|
*,
|
||||||
|
'create table if not exists ' || snk_db || '.' || snk_tbl || ' as ' || chr(10) ||
|
||||||
|
'select distinct ' || col_with_alias || chr(10) ||
|
||||||
|
'from ' || src_db || '.' || src_tbl || chr(10) ||
|
||||||
|
'where COALESCE(' || src_cols || ', null) is not null; ' || chr(10) || chr(10) ||
|
||||||
|
|
||||||
|
'delete from ' || snk_db || '.' || snk_tbl || chr(10) ||
|
||||||
|
' where ' || prep_key || chr(10) ||
|
||||||
|
'in (select ' || prep_key || chr(10) || 'from ' || snk_db || '.' || snk_tbl || chr(10) ||
|
||||||
|
'except ' || chr(10) ||
|
||||||
|
'select ' || src_key || chr(10) || 'from ' || src_db || '.' || src_tbl || chr(10) ||
|
||||||
|
'); ' || chr(10) || chr(10) ||
|
||||||
|
|
||||||
|
'delete from ' || snk_db || '.' || unif_input_tbl || chr(10) ||
|
||||||
|
' where ' || prep_key || chr(10) ||
|
||||||
|
'in (select ' || prep_key || chr(10) || 'from ' || snk_db || '.' || unif_input_tbl || chr(10) || ' where source = ''''' || src_tbl || ''''' ' || chr(10) ||
|
||||||
|
'except ' || chr(10) ||
|
||||||
|
'select ' || prep_key || chr(10) || 'from ' || src_db || '.' || snk_tbl || chr(10) ||
|
||||||
|
')
|
||||||
|
and source = ''''' || src_tbl || ''''' ; ' || chr(10) || chr(10) ||
|
||||||
|
|
||||||
|
|
||||||
|
'insert into ' || snk_db || '.' || snk_tbl || chr(10) ||
|
||||||
|
'with new_records as (' || chr(10) ||
|
||||||
|
'select ' || col_with_alias || chr(10) || 'from ' || src_db || '.' || src_tbl || chr(10) ||
|
||||||
|
'except ' || chr(10) ||
|
||||||
|
'select ' || prep_cols || chr(10) || 'from ' || snk_db || '.' || snk_tbl || chr(10) ||
|
||||||
|
')
|
||||||
|
select *
|
||||||
|
, TD_TIME_PARSE(cast(CURRENT_TIMESTAMP as varchar)) as time
|
||||||
|
from new_records
|
||||||
|
where COALESCE(' || prep_cols || ', null) is not null;'
|
||||||
|
|
||||||
|
as prep_tbl_sql_string,
|
||||||
|
|
||||||
|
'insert into ' || snk_db || '.' || unif_input_tbl || chr(10) ||
|
||||||
|
'select ' || prep_cols || ', time, ''''' || src_tbl || ''''' as source, time as ingest_time' || chr(10) || 'from ' || snk_db || '.' || snk_tbl || chr(10) ||
|
||||||
|
'where time > (' || chr(10) || ' select coalesce(max(time), 0) from ' || snk_db || '.' || unif_input_tbl || chr(10) || ' where source = ''''' || src_tbl || '''''' || chr(10) || ');'
|
||||||
|
as unif_input_tbl_sql_string
|
||||||
|
|
||||||
|
from flaten_data_agg
|
||||||
|
)
|
||||||
|
select *
|
||||||
|
from prep_table_sqls
|
||||||
|
order by 1, 2, 3, 4
|
||||||
|
```
|
||||||
|
|
||||||
|
### 4. unification/queries/unif_input_tbl.sql (Queries Directory)
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - DSAR EXCLUSION & DATA PROCESSING ⚠️**
|
||||||
|
**Production DSAR processing and data cleaning - DO NOT MODIFY ANY PART:**
|
||||||
|
**⚠️ CRITICAL, ONLY ADD THE COLUMNS IN data_cleaned CTE {List columns other than email and phone from alias_as src_prep_params.yml file}**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
|
||||||
|
---- Storing DSAR Masked values into exclusion_list.
|
||||||
|
insert into ${client_short_name}_${lkup}.exclusion_list
|
||||||
|
with dsar_masked as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
'phone' as key_name,
|
||||||
|
phone as key_value
|
||||||
|
from ${client_short_name}_${stg}.${globals.unif_input_tbl}_tmp_td
|
||||||
|
where (LENGTH(phone) = 64 or LENGTH(phone) > 10 )
|
||||||
|
)
|
||||||
|
select
|
||||||
|
key_value,
|
||||||
|
key_name,
|
||||||
|
ARRAY['${client_short_name}_${stg}.${globals.unif_input_tbl}_tmp_td'] as tbls,
|
||||||
|
'DSAR masked phone' as note
|
||||||
|
from dsar_masked
|
||||||
|
where key_value not in (
|
||||||
|
select key_value from ${client_short_name}_${lkup}.exclusion_list
|
||||||
|
where key_name = 'phone'
|
||||||
|
and nullif(key_value, '') is not null
|
||||||
|
)
|
||||||
|
group by 1, 2;
|
||||||
|
|
||||||
|
---- Storing DSAR Masked values into exclusion_list.
|
||||||
|
insert into ${client_short_name}_${lkup}.exclusion_list
|
||||||
|
with dsar_masked as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
'email' as key_name,
|
||||||
|
email as key_value
|
||||||
|
from ${client_short_name}_${stg}.${globals.unif_input_tbl}_tmp_td
|
||||||
|
where (LENGTH(email) = 64 and email not like '%@%')
|
||||||
|
)
|
||||||
|
select
|
||||||
|
key_value,
|
||||||
|
key_name,
|
||||||
|
ARRAY['${client_short_name}_${stg}.${globals.unif_input_tbl}_tmp_td'] as tbls,
|
||||||
|
'DSAR masked email' as note
|
||||||
|
from dsar_masked
|
||||||
|
where key_value not in (
|
||||||
|
select key_value from ${client_short_name}_${lkup}.exclusion_list
|
||||||
|
where key_name = 'email'
|
||||||
|
and nullif(key_value, '') is not null
|
||||||
|
)
|
||||||
|
group by 1, 2;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
drop table if exists ${client_short_name}_${stg}.${globals.unif_input_tbl};
|
||||||
|
create table if not exists ${client_short_name}_${stg}.${globals.unif_input_tbl} (time bigint);
|
||||||
|
|
||||||
|
insert into ${client_short_name}_${stg}.${globals.unif_input_tbl}
|
||||||
|
with get_latest_data as
|
||||||
|
(
|
||||||
|
select *
|
||||||
|
from ${client_short_name}_${stg}.${globals.unif_input_tbl}_tmp_td a
|
||||||
|
where a.time > (
|
||||||
|
select COALESCE(max(time), 0) from ${client_short_name}_${stg}.${globals.unif_input_tbl}
|
||||||
|
)
|
||||||
|
)
|
||||||
|
, data_cleaned as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
-- **AUTOMATIC COLUMN DETECTION** - Agent will query schema and insert columns here
|
||||||
|
-- The dynamic-prep-creation agent will:
|
||||||
|
-- 1. Query: SELECT column_name FROM information_schema.columns
|
||||||
|
-- WHERE table_schema = '${client_short_name}_${stg}'
|
||||||
|
-- AND table_name = '${globals.unif_input_tbl}_tmp_td'
|
||||||
|
-- AND column_name NOT IN ('email', 'phone', 'source', 'ingest_time', 'time')
|
||||||
|
-- ORDER BY ordinal_position
|
||||||
|
-- 2. Generate: a.column_name, for each remaining column
|
||||||
|
-- 3. Insert the column list here automatically
|
||||||
|
-- **AGENT_DYNAMIC_COLUMNS_PLACEHOLDER** -- Do not remove this comment
|
||||||
|
case when e.key_value is null then a.email else null end email,
|
||||||
|
case when p.key_value is null then a.phone else null end phone,
|
||||||
|
a.source,
|
||||||
|
a.ingest_time,
|
||||||
|
a.time
|
||||||
|
from get_latest_data a
|
||||||
|
left join ${client_short_name}_${lkup}.exclusion_list e on a.email = e.key_value and e.key_name = 'email'
|
||||||
|
left join ${client_short_name}_${lkup}.exclusion_list p on a.phone = p.key_value and p.key_name = 'phone'
|
||||||
|
)
|
||||||
|
select
|
||||||
|
*
|
||||||
|
from data_cleaned
|
||||||
|
where coalesce(email, phone) is not null
|
||||||
|
;
|
||||||
|
|
||||||
|
-- set session join_distribution_type = 'BROADCAST'
|
||||||
|
-- set session time_partitioning_range = 'none'
|
||||||
|
|
||||||
|
-- drop table if exists ${client_short_name}_${stg}.work_${globals.unif_input_tbl};
|
||||||
|
```
|
||||||
|
|
||||||
|
## Dynamic File to Create (Based on Main Agent Analysis)
|
||||||
|
|
||||||
|
### 5. unification/config/environment.yml (Config Directory)
|
||||||
|
**⚠️ STRUCTURE CRITICAL: MUST be created EXACTLY AS IS - PRODUCTION VARIABLES ⚠️**
|
||||||
|
**Required for variable definitions - DO NOT MODIFY STRUCTURE:**
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
# Client and environment configuration
|
||||||
|
client_short_name: client_name # Replace with actual client short name
|
||||||
|
src: src # Source database suffix
|
||||||
|
stg: stg # Staging database suffix
|
||||||
|
gld: gld
|
||||||
|
lkup: references
|
||||||
|
```
|
||||||
|
|
||||||
|
### 6. unification/config/src_prep_params.yml (Config Directory)
|
||||||
|
Create this file based on the main agent's table analysis. Follow the EXACT structure from src_prep_params_example.yml:
|
||||||
|
|
||||||
|
**Structure Requirements:**
|
||||||
|
- `globals:` section with `unif_input_tbl: unif_input`
|
||||||
|
- `prep_tbls:` section containing array of table configurations
|
||||||
|
- Each table must have: `src_tbl`, `src_db`, `snk_db`, `snk_tbl`, `columns`
|
||||||
|
- Each column must have: `name` (source column) and `alias_as` (unified alias)
|
||||||
|
|
||||||
|
**Column Alias Standards:**
|
||||||
|
- Email columns → `alias_as: email`
|
||||||
|
- Phone columns → `alias_as: phone`
|
||||||
|
- Loyalty ID columns → `alias_as: loyalty_id`
|
||||||
|
- Customer ID columns → `alias_as: customer_id`
|
||||||
|
- Credit card columns → `alias_as: credit_card_token`
|
||||||
|
- TD Client ID columns → `alias_as: td_client_id`
|
||||||
|
- TD Global ID columns → `alias_as: td_global_id`
|
||||||
|
|
||||||
|
**⚠️ CRITICAL: DO NOT ADD TIME COLUMN ⚠️**
|
||||||
|
- **NEVER ADD** `time` column to src_prep_params.yml columns list
|
||||||
|
- **TIME IS AUTO-GENERATED** by the generic SQL template at line 66: `TD_TIME_PARSE(cast(CURRENT_TIMESTAMP as varchar)) as time`
|
||||||
|
- **ONLY INCLUDE** actual identifier columns from table analysis
|
||||||
|
- **TIME COLUMN** is automatically added by production SQL and used for incremental processing
|
||||||
|
|
||||||
|
**Example Structure:**
|
||||||
|
```yaml
|
||||||
|
globals:
|
||||||
|
unif_input_tbl: unif_input
|
||||||
|
|
||||||
|
prep_tbls:
|
||||||
|
- src_tbl: table_name
|
||||||
|
src_db: ${client_short_name}_${stg}
|
||||||
|
snk_db: ${client_short_name}_${stg}
|
||||||
|
snk_tbl: ${src_tbl}_prep
|
||||||
|
columns:
|
||||||
|
- col:
|
||||||
|
name: source_column_name
|
||||||
|
alias_as: unified_alias_name
|
||||||
|
```
|
||||||
|
|
||||||
|
## Agent Workflow
|
||||||
|
|
||||||
|
### When Called by Main Agent:
|
||||||
|
1. **Create directory structure first** unification/config/, unification/queries/)
|
||||||
|
2. **Always create the 4 generic files** (dynmic_prep_creation.dig, create_schema.sql, loop_on_tables.sql, unif_input_tbl.sql)
|
||||||
|
3. **Create environment.yml** with client configuration
|
||||||
|
4. **Analyze provided table information** from main agent
|
||||||
|
5. **Create src_prep_params.yml** based on analysis following exact structure
|
||||||
|
6. **🚨 CRITICAL: DYNAMIC COLUMN DETECTION** for unif_input_tbl.sql:
|
||||||
|
- **MUST QUERY**: `SELECT column_name FROM information_schema.columns WHERE table_schema = '{client_stg_db}' AND table_name = '{unif_input_tbl}_tmp_td' AND column_name NOT IN ('email', 'phone', 'source', 'ingest_time', 'time') ORDER BY ordinal_position`
|
||||||
|
- **MUST REPLACE**: `-- **AGENT_DYNAMIC_COLUMNS_PLACEHOLDER** -- Do not remove this comment`
|
||||||
|
- **WITH**: `a.column1, a.column2, a.column3,` (for each remaining column)
|
||||||
|
- **FORMAT**: Each column as `a.{column_name},` with proper trailing comma
|
||||||
|
- **EXAMPLE**: If remaining columns are [customer_id, user_id, profile_id], insert: `a.customer_id, a.user_id, a.profile_id,`
|
||||||
|
7. **Validate all files** are created correctly
|
||||||
|
|
||||||
|
### Critical Requirements:
|
||||||
|
- **⚠️ NEVER MODIFY THE 5 GENERIC FILES ⚠️** - they must be created EXACTLY AS IS
|
||||||
|
- **EXACT FILENAME**: `dynmic_prep_creation.dig`
|
||||||
|
- **EXACT CONTENT**: Every character, space, variable must match specifications
|
||||||
|
- **EXACT STRUCTURE**: No changes to YAML structure, SQL logic, or variable names
|
||||||
|
- **Maintain exact YAML structure** in src_prep_params.yml
|
||||||
|
- **Use standard column aliases** for unification compatibility
|
||||||
|
- **Preserve variable placeholders** like `${client_short_name}_${stg}`
|
||||||
|
- **Create queries directory** if it doesn't exist
|
||||||
|
- **Create config directory** if it doesn't exist
|
||||||
|
|
||||||
|
### ⚠️ FAILURE PREVENTION ⚠️
|
||||||
|
- **CHECK FILENAME**: Verify "dynmic_prep_creation.dig" (NO 'a' in dynamic)
|
||||||
|
- **COPY EXACT CONTENT**: Use Write tool with EXACT text from instructions
|
||||||
|
- **NO CREATIVE CHANGES**: Do not improve, optimize, or modify any part
|
||||||
|
- **VALIDATE OUTPUT**: Ensure every file matches the template exactly
|
||||||
|
|
||||||
|
### File Paths (EXACT NAMES REQUIRED):
|
||||||
|
- `unification/config/` directory (create if missing)
|
||||||
|
- `unification/queries/` directory (create if missing)
|
||||||
|
- `unification/dynmic_prep_creation.dig` (root directory) **⚠️ NO 'a' in dynmic ⚠️**
|
||||||
|
- `unification/queries/create_schema.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/queries/loop_on_tables.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/config/environment.yml` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/config/src_prep_params.yml` (dynamic based on analysis)
|
||||||
|
- `unification/queries/unif_input_tbl.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
|
||||||
|
## Error Prevention & Validation:
|
||||||
|
- **MANDATORY VALIDATION**: After creating each generic file, verify it matches the template EXACTLY
|
||||||
|
- **EXACT FILENAME CHECK**: Confirm "dynmic_prep_creation.dig"
|
||||||
|
- **CONTENT VERIFICATION**: Every line, space, variable must match the specification
|
||||||
|
- **NO IMPROVEMENTS**: Do not add comments, change formatting, or optimize anything
|
||||||
|
- **Always use Write tool** to create files with exact content
|
||||||
|
- **Never modify generic SQL or DIG content** under any circumstances
|
||||||
|
- **Ensure directory structure** is created before writing files
|
||||||
|
- **Validate YAML syntax** in src_prep_params.yml
|
||||||
|
- **Follow exact indentation** and formatting from examples
|
||||||
|
|
||||||
|
## 🚨 DYNAMIC COLUMN DETECTION IMPLEMENTATION 🚨
|
||||||
|
|
||||||
|
### **OPTIMIZATION BENEFITS:**
|
||||||
|
- **🎯 AUTOMATIC**: No manual column management required
|
||||||
|
- **🔄 FLEXIBLE**: Adapts to schema changes automatically
|
||||||
|
- **🛠️ FUTURE-PROOF**: Works when new columns are added to unified table
|
||||||
|
- **❌ NO ERRORS**: Eliminates "column not found" issues
|
||||||
|
- **⚡ OPTIMAL**: Uses only necessary columns, avoids SELECT *
|
||||||
|
- **🔒 SECURE**: Properly handles email/phone exclusion logic
|
||||||
|
|
||||||
|
### **PROBLEM SOLVED:**
|
||||||
|
- **BEFORE**: Manual column listing → breaks when schema changes
|
||||||
|
- **AFTER**: Dynamic detection → automatically adapts to any schema changes
|
||||||
|
|
||||||
|
### MANDATORY STEP-BY-STEP PROCESS FOR unif_input_tbl.sql:
|
||||||
|
|
||||||
|
1. **AFTER creating the base unif_input_tbl.sql file**, perform dynamic column detection:
|
||||||
|
2. **QUERY SCHEMA**: Use MCP TD tools to execute:
|
||||||
|
```sql
|
||||||
|
SELECT column_name
|
||||||
|
FROM information_schema.columns
|
||||||
|
WHERE table_schema = '{client_short_name}_stg'
|
||||||
|
AND table_name = '{unif_input_tbl}_tmp_td'
|
||||||
|
AND column_name NOT IN ('email', 'phone', 'source', 'ingest_time', 'time')
|
||||||
|
ORDER BY ordinal_position
|
||||||
|
```
|
||||||
|
3. **EXTRACT RESULTS**: Get list of remaining columns (e.g., ['customer_id', 'user_id', 'profile_id'])
|
||||||
|
4. **FORMAT COLUMNS**: Create string like: `a.customer_id, a.user_id, a.profile_id,`
|
||||||
|
5. **LOCATE PLACEHOLDER**: Find line with `-- **AGENT_DYNAMIC_COLUMNS_PLACEHOLDER** -- Do not remove this comment`
|
||||||
|
6. **REPLACE PLACEHOLDER**: Replace the placeholder line with the formatted column list
|
||||||
|
7. **VERIFY SYNTAX**: Ensure proper comma placement and SQL syntax
|
||||||
|
|
||||||
|
### EXAMPLE TRANSFORMATION:
|
||||||
|
**BEFORE (placeholder):**
|
||||||
|
```sql
|
||||||
|
-- **AGENT_DYNAMIC_COLUMNS_PLACEHOLDER** -- Do not remove this comment
|
||||||
|
case when e.key_value is null then a.email else null end email,
|
||||||
|
```
|
||||||
|
|
||||||
|
**AFTER (dynamic columns inserted):**
|
||||||
|
```sql
|
||||||
|
a.customer_id, a.user_id, a.profile_id,
|
||||||
|
case when e.key_value is null then a.email else null end email,
|
||||||
|
```
|
||||||
|
|
||||||
|
## ⚠️ CRITICAL SUCCESS CRITERIA ⚠️
|
||||||
|
1. ALL FILES MUST BE CREATED UNDER unification/ directory.
|
||||||
|
1.1 File named "dynmic_prep_creation.dig" exists
|
||||||
|
1.2 File named "unif_input_tbl.sql" exists with EXACT SQL content
|
||||||
|
2. Content matches template character-for-character
|
||||||
|
3. All variable placeholders preserved exactly
|
||||||
|
4. No additional comments or modifications
|
||||||
|
5. Queries folder contains exact SQL files (create_schema.sql, loop_on_tables.sql, unif_input_tbl.sql)
|
||||||
|
6. Config folder contains exact YAML files
|
||||||
|
7. **🚨 DYNAMIC COLUMNS**: unif_input_tbl.sql MUST have placeholder replaced with actual columns
|
||||||
|
8. **🚨 SCHEMA QUERY**: Agent MUST query information_schema to get remaining columns
|
||||||
|
9. **🚨 SYNTAX VALIDATION**: Final SQL MUST be syntactically correct with proper commas
|
||||||
|
|
||||||
|
**FAILURE TO MEET ANY CRITERIA = BROKEN PRODUCTION SYSTEM**
|
||||||
268
agents/id-unification-creator.md
Normal file
268
agents/id-unification-creator.md
Normal file
@@ -0,0 +1,268 @@
|
|||||||
|
---
|
||||||
|
name: id-unification-creator
|
||||||
|
description: Creates core ID unification configuration files (unify.yml and id_unification.dig) based on completed prep analysis and user requirements
|
||||||
|
model: sonnet
|
||||||
|
color: yellow
|
||||||
|
---
|
||||||
|
|
||||||
|
# ID Unification Creator Sub-Agent
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Create core ID unification configuration files (unify.yml and id_unification.dig) based on completed prep table analysis and user requirements.
|
||||||
|
|
||||||
|
**CRITICAL**: This sub-agent ONLY creates the core unification files. It does NOT create prep files, enrichment files, or orchestration workflows - those are handled by other specialized sub-agents.
|
||||||
|
|
||||||
|
## Input Requirements
|
||||||
|
The main agent will provide:
|
||||||
|
- **Key Analysis Results**: Finalized key columns and mappings from unif-keys-extractor
|
||||||
|
- **Prep Configuration**: Completed prep table configuration (config/src_prep_params.yml must exist)
|
||||||
|
- **User Selections**: ID method (persistent_id vs canonical_id), update method (full refresh vs incremental), region, client details
|
||||||
|
- **Environment Setup**: Client configuration (config/environment.yml must exist)
|
||||||
|
|
||||||
|
## Core Responsibilities
|
||||||
|
|
||||||
|
### 1. Create unify.yml Configuration
|
||||||
|
Generate complete YAML configuration with:
|
||||||
|
- **keys** section with validation patterns
|
||||||
|
- **tables** section referencing unified prep table only
|
||||||
|
- **Method-specific ID configuration** (persistent_ids OR canonical_ids, never both)
|
||||||
|
- **Dynamic key mappings** based on actual prep analysis
|
||||||
|
- **Variable references**: Uses ${globals.unif_input_tbl} and ${client_short_name}_${stg}
|
||||||
|
|
||||||
|
### 2. Create id_unification.dig Workflow
|
||||||
|
Generate core unification workflow with:
|
||||||
|
- **Regional endpoint** based on user selection
|
||||||
|
- **Method flags** (only the selected method enabled)
|
||||||
|
- **Authentication** using TD secret format
|
||||||
|
- **HTTP API call** to TD unification service
|
||||||
|
- **⚠️ CRITICAL**: Must include BOTH config files in _export to resolve variables in unify.yml
|
||||||
|
|
||||||
|
### 3. Schema Validation & Update (CRITICAL)
|
||||||
|
Prevent first-run failures by ensuring schema completeness:
|
||||||
|
- **Read unify.yml**: Extract complete merge_by_keys list
|
||||||
|
- **Read create_schema.sql**: Check existing column definitions
|
||||||
|
- **Compare & Update**: Add any missing columns from merge_by_keys to schema
|
||||||
|
- **Required columns**: All merge_by_keys + source, time, ingest_time
|
||||||
|
- **Update both tables**: ${globals.unif_input_tbl} AND ${globals.unif_input_tbl}_tmp_td
|
||||||
|
|
||||||
|
## Critical Configuration Requirements
|
||||||
|
|
||||||
|
### Regional Endpoints (MUST use correct endpoint)
|
||||||
|
1. **US** - `https://api-cdp.treasuredata.com/unifications/workflow_call`
|
||||||
|
2. **EU** - `https://api-cdp.eu01.treasuredata.com/unifications/workflow_call`
|
||||||
|
3. **Asia Pacific** - `https://api-cdp.ap02.treasuredata.com/unifications/workflow_call`
|
||||||
|
4. **Japan** - `https://api-cdp.treasuredata.co.jp/unifications/workflow_call`
|
||||||
|
|
||||||
|
### unify.yml Template Structure
|
||||||
|
```yaml
|
||||||
|
name: {unif_name}
|
||||||
|
|
||||||
|
keys:
|
||||||
|
- name: email
|
||||||
|
invalid_texts: ['']
|
||||||
|
- name: td_client_id
|
||||||
|
invalid_texts: ['']
|
||||||
|
- name: phone
|
||||||
|
invalid_texts: ['']
|
||||||
|
- name: td_global_id
|
||||||
|
invalid_texts: ['']
|
||||||
|
# ADD OTHER DYNAMIC KEYS from prep analysis
|
||||||
|
|
||||||
|
tables:
|
||||||
|
- database: ${client_short_name}_${stg}
|
||||||
|
table: ${globals.unif_input_tbl}
|
||||||
|
incremental_columns: [time]
|
||||||
|
key_columns:
|
||||||
|
# USE ALL alias_as columns from prep configuration
|
||||||
|
- {column: email, key: email}
|
||||||
|
- {column: phone, key: phone}
|
||||||
|
- {column: td_client_id, key: td_client_id}
|
||||||
|
- {column: td_global_id, key: td_global_id}
|
||||||
|
# ADD OTHER DYNAMIC KEY MAPPINGS
|
||||||
|
|
||||||
|
# Choose EITHER canonical_ids OR persistent_ids (NEVER both)
|
||||||
|
persistent_ids:
|
||||||
|
- name: {persistent_id_name}
|
||||||
|
merge_by_keys: [email, td_client_id, phone, td_global_id] # ALL available keys
|
||||||
|
merge_iterations: 15
|
||||||
|
|
||||||
|
canonical_ids:
|
||||||
|
- name: {canonical_id_name}
|
||||||
|
merge_by_keys: [email, td_client_id, phone, td_global_id] # ALL available keys
|
||||||
|
merge_iterations: 15
|
||||||
|
```
|
||||||
|
|
||||||
|
### unification/id_unification.dig Template Structure
|
||||||
|
```yaml
|
||||||
|
timezone: UTC
|
||||||
|
|
||||||
|
_export:
|
||||||
|
!include : config/environment.yml
|
||||||
|
!include : config/src_prep_params.yml
|
||||||
|
|
||||||
|
+call_unification:
|
||||||
|
http_call>: {REGIONAL_ENDPOINT_URL}
|
||||||
|
headers:
|
||||||
|
- authorization: ${secret:td.apikey}
|
||||||
|
- content-type: application/json
|
||||||
|
method: POST
|
||||||
|
retry: true
|
||||||
|
content_format: json
|
||||||
|
content:
|
||||||
|
run_persistent_ids: {true/false} # ONLY if persistent_id selected
|
||||||
|
run_canonical_ids: {true/false} # ONLY if canonical_id selected
|
||||||
|
run_enrichments: true # ALWAYS true
|
||||||
|
run_master_tables: true # ALWAYS true
|
||||||
|
full_refresh: {true/false} # Based on user selection
|
||||||
|
keep_debug_tables: true # ALWAYS true
|
||||||
|
unification:
|
||||||
|
!include : config/unify.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
## Dynamic Configuration Logic
|
||||||
|
|
||||||
|
### Key Detection and Mapping
|
||||||
|
1. **Read Prep Configuration**: Parse config/src_prep_params.yml to get all alias_as columns
|
||||||
|
2. **Extract Available Keys**: Identify all unique key types from prep table mappings
|
||||||
|
3. **Generate keys Section**: Create validation rules for each detected key type
|
||||||
|
4. **Generate key_columns**: Map each alias_as column to its corresponding key type
|
||||||
|
5. **Generate merge_by_keys**: Include ALL available key types in the merge list
|
||||||
|
|
||||||
|
### Method-Specific Configuration
|
||||||
|
- **persistent_ids method**:
|
||||||
|
- Include `persistent_ids:` section with user-specified name
|
||||||
|
- Set `run_persistent_ids: true` in workflow
|
||||||
|
- Do NOT include `canonical_ids:` section
|
||||||
|
- Do NOT set `run_canonical_ids` flag
|
||||||
|
|
||||||
|
- **canonical_ids method**:
|
||||||
|
- Include `canonical_ids:` section with user-specified name
|
||||||
|
- Set `run_canonical_ids: true` in workflow
|
||||||
|
- Do NOT include `persistent_ids:` section
|
||||||
|
- Do NOT set `run_persistent_ids` flag
|
||||||
|
|
||||||
|
### Update Method Configuration
|
||||||
|
- **Full Refresh**: Set `full_refresh: true` in workflow
|
||||||
|
- **Incremental**: Set `full_refresh: false` in workflow
|
||||||
|
|
||||||
|
## Implementation Instructions
|
||||||
|
|
||||||
|
**⚠️ MANDATORY**: Follow interactive configuration pattern from `/plugins/INTERACTIVE_CONFIG_GUIDE.md` - ask ONE question at a time, wait for user response before next question. See guide for complete list of required parameters.
|
||||||
|
|
||||||
|
### Step 1: Validate Prerequisites
|
||||||
|
```
|
||||||
|
ENSURE the following files exist before proceeding:
|
||||||
|
- config/environment.yml (client configuration)
|
||||||
|
- config/src_prep_params.yml (prep table configuration)
|
||||||
|
|
||||||
|
READ both files to extract:
|
||||||
|
- client_short_name (from environment.yml)
|
||||||
|
- globals.unif_input_tbl (from src_prep_params.yml)
|
||||||
|
- All prep_tbls with alias_as mappings (from src_prep_params.yml)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 2: Extract Key Information
|
||||||
|
```
|
||||||
|
PARSE config/src_prep_params.yml to identify:
|
||||||
|
- All unique alias_as column names across all prep tables
|
||||||
|
- Key types present: email, phone, td_client_id, td_global_id, customer_id, user_id, etc.
|
||||||
|
- Generate complete list of available keys for merge_by_keys
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 3: Generate unification/unify.yml
|
||||||
|
```
|
||||||
|
CREATE unification/config/unify.yml with:
|
||||||
|
- name: {user_provided_unif_name}
|
||||||
|
- keys: section with ALL detected key types and their validation patterns
|
||||||
|
- tables: section with SINGLE table reference (${globals.unif_input_tbl})
|
||||||
|
- key_columns: ALL alias_as columns mapped to their key types
|
||||||
|
- Method section: EITHER persistent_ids OR canonical_ids (never both)
|
||||||
|
- merge_by_keys: ALL available key types in priority order
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 4: Validate and Update Schema
|
||||||
|
```
|
||||||
|
CRITICAL SCHEMA VALIDATION - Prevent First Run Failures:
|
||||||
|
|
||||||
|
1. READ unification/config/unify.yml to extract merge_by_keys list
|
||||||
|
2. READ unification/queries/create_schema.sql to check existing columns
|
||||||
|
3. COMPARE required columns vs existing columns:
|
||||||
|
- Required: All keys from merge_by_keys list + source, time, ingest_time
|
||||||
|
- Existing: Parse CREATE TABLE statements to find current columns
|
||||||
|
4. UPDATE create_schema.sql if missing columns:
|
||||||
|
- Add missing columns as "varchar" data type
|
||||||
|
- Preserve existing structure and variable placeholders
|
||||||
|
- Update BOTH table definitions (${globals.unif_input_tbl} AND ${globals.unif_input_tbl}_tmp_td)
|
||||||
|
|
||||||
|
EXAMPLE: If merge_by_keys contains [email, customer_id, user_id] but create_schema.sql only has "source varchar":
|
||||||
|
- Add: email varchar, customer_id varchar, user_id varchar, time bigint, ingest_time bigint
|
||||||
|
- Result: Complete schema with all required columns for successful first run
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 5: Generate unification/id_unification.dig
|
||||||
|
```
|
||||||
|
CREATE unification/id_unification.dig with:
|
||||||
|
- timezone: UTC
|
||||||
|
- _export:
|
||||||
|
!include : config/environment.yml # For ${client_short_name}, ${stg}
|
||||||
|
!include : config/src_prep_params.yml # For ${globals.unif_input_tbl}
|
||||||
|
- http_call: correct regional endpoint URL
|
||||||
|
- headers: authorization and content-type
|
||||||
|
- Method flags: ONLY the selected method enabled
|
||||||
|
- full_refresh: based on user selection
|
||||||
|
- unification: !include : config/unify.yml
|
||||||
|
|
||||||
|
⚠️ BOTH config files are REQUIRED because unify.yml contains variables from both:
|
||||||
|
- ${client_short_name}_${stg} (from environment.yml)
|
||||||
|
- ${globals.unif_input_tbl} (from src_prep_params.yml)
|
||||||
|
```
|
||||||
|
|
||||||
|
## File Output Specifications
|
||||||
|
|
||||||
|
### File Locations
|
||||||
|
- **unify.yml**: `unification/config/unify.yml` (relative to project root)
|
||||||
|
- **id_unification.dig**: `unification/id_unification.dig` (project root)
|
||||||
|
|
||||||
|
### Critical Requirements
|
||||||
|
- **NO master_tables section**: Handled automatically by TD
|
||||||
|
- **Single table reference**: Use ${globals.unif_input_tbl} only
|
||||||
|
- **All available keys**: Include every key type found in prep configuration
|
||||||
|
- **Exact template format**: Follow TD-compliant YAML/DIG syntax
|
||||||
|
- **Dynamic variable replacement**: Use actual values from prep analysis
|
||||||
|
- **Method exclusivity**: Never include both persistent_ids AND canonical_ids
|
||||||
|
|
||||||
|
## Error Prevention
|
||||||
|
|
||||||
|
### Common Issues to Avoid
|
||||||
|
- **Missing content-type header**: MUST include both authorization AND content-type
|
||||||
|
- **Wrong endpoint region**: Use exact URL based on user selection
|
||||||
|
- **Multiple ID methods**: Include ONLY the selected method
|
||||||
|
- **Missing key validations**: All keys must have invalid_texts, UUID keys need valid_regexp
|
||||||
|
- **Prep table mismatch**: Key mappings must match alias_as columns exactly
|
||||||
|
- **⚠️ CRITICAL: Schema mismatch**: create_schema.sql MUST contain ALL columns from merge_by_keys list
|
||||||
|
- **⚠️ CRITICAL: Incomplete _export section**: MUST include BOTH config/environment.yml AND config/src_prep_params.yml in _export section
|
||||||
|
|
||||||
|
### Validation Checklist
|
||||||
|
Before completing:
|
||||||
|
- [ ] unify.yml contains all detected key types from prep analysis
|
||||||
|
- [ ] key_columns section maps ALL alias_as columns
|
||||||
|
- [ ] Only ONE ID method section exists (persistent_ids OR canonical_ids)
|
||||||
|
- [ ] merge_by_keys includes ALL available keys
|
||||||
|
- [ ] **CRITICAL SCHEMA**: create_schema.sql contains ALL columns from merge_by_keys list
|
||||||
|
- [ ] **CRITICAL SCHEMA**: Both table definitions updated with required columns (${globals.unif_input_tbl} AND ${globals.unif_input_tbl}_tmp_td)
|
||||||
|
- [ ] id_unification.dig has correct regional endpoint
|
||||||
|
- [ ] **CRITICAL**: id_unification.dig _export section includes BOTH config/environment.yml AND config/src_prep_params.yml
|
||||||
|
- [ ] Workflow flags match selected method only
|
||||||
|
- [ ] Both files use proper TD YAML/DIG syntax
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
- ALL FILES MUST BE CREATED UNDER `unification/` directory.
|
||||||
|
- **TD-Compliant Output**: Files work without modification in TD
|
||||||
|
- **Dynamic Configuration**: Based on actual prep analysis, not hardcoded
|
||||||
|
- **Method Accuracy**: Exact implementation of user selections
|
||||||
|
- **Regional Correctness**: Proper endpoint for user's region
|
||||||
|
- **Key Completeness**: All identified keys included with proper validation
|
||||||
|
- **⚠️ CRITICAL: Schema Completeness**: create_schema.sql contains ALL columns from merge_by_keys to prevent first-run failures
|
||||||
|
- **Template Fidelity**: Exact format matching TD requirements
|
||||||
|
|
||||||
|
**IMPORTANT**: This sub-agent creates ONLY the core unification files. The main agent handles orchestration, prep creation, and enrichment through other specialized sub-agents.
|
||||||
261
agents/unif-keys-extractor.md
Normal file
261
agents/unif-keys-extractor.md
Normal file
@@ -0,0 +1,261 @@
|
|||||||
|
---
|
||||||
|
name: unif-keys-extractor
|
||||||
|
description: STRICT user identifier extraction agent that ONLY includes tables with PII/user data using REAL Treasure Data analysis. ZERO TOLERANCE for guessing or including non-PII tables.
|
||||||
|
model: sonnet
|
||||||
|
color: purple
|
||||||
|
---
|
||||||
|
|
||||||
|
# 🚨 UNIF-KEYS-EXTRACTOR - ZERO-TOLERANCE PII EXTRACTION AGENT 🚨
|
||||||
|
|
||||||
|
## CRITICAL MANDATE - NO EXCEPTIONS
|
||||||
|
**THIS AGENT OPERATES UNDER ZERO-TOLERANCE POLICY:**
|
||||||
|
- ❌ **NO GUESSING** column names or data patterns
|
||||||
|
- ❌ **NO INCLUDING** tables without user identifiers
|
||||||
|
- ❌ **NO ASSUMPTIONS** about table contents
|
||||||
|
- ✅ **ONLY REAL DATA** from Treasure Data MCP tools
|
||||||
|
- ✅ **ONLY PII TABLES** that contain actual user identifiers
|
||||||
|
- ✅ **MANDATORY VALIDATION** at every step
|
||||||
|
|
||||||
|
**⚠️ MANDATORY**: Follow interactive configuration pattern from `/plugins/INTERACTIVE_CONFIG_GUIDE.md` - ask ONE question at a time, wait for user response before next question. See guide for complete list of required parameters.
|
||||||
|
|
||||||
|
## 🔴 CRYSTAL CLEAR USER IDENTIFIER DEFINITION 🔴
|
||||||
|
|
||||||
|
### ✅ VALID USER IDENTIFIERS (MUST BE PRESENT TO INCLUDE TABLE)
|
||||||
|
**A table MUST contain AT LEAST ONE of these column types to be included:**
|
||||||
|
|
||||||
|
#### **PRIMARY USER IDENTIFIERS:**
|
||||||
|
- **Email columns**: `email`, `email_std`, `email_address`, `email_address_std`, `user_email`, `customer_email`, `recipient_email`, `recipient_email_std`
|
||||||
|
- **Phone columns**: `phone`, `phone_std`, `phone_number`, `mobile_phone`, `customer_phone`
|
||||||
|
- **User ID columns**: `user_id`, `customer_id`, `account_id`, `member_id`, `uid`, `user_uuid`
|
||||||
|
- **Identity columns**: `profile_id`, `identity_id`, `cognito_identity_userid`, `flavormaker_uid`
|
||||||
|
- **Cookie/Device IDs**: `td_client_id`, `td_global_id`, `td_ssc_id`, `cookie_id`, `device_id`
|
||||||
|
|
||||||
|
### ❌ NOT USER IDENTIFIERS (EXCLUDE TABLES WITH ONLY THESE)
|
||||||
|
**These columns DO NOT qualify as user identifiers:**
|
||||||
|
|
||||||
|
#### **SYSTEM/METADATA COLUMNS:**
|
||||||
|
- `id`, `created_at`, `updated_at`, `load_timestamp`, `source_system`, `time`
|
||||||
|
|
||||||
|
#### **CAMPAIGN/MARKETING COLUMNS:**
|
||||||
|
- `campaign_id`, `campaign_name`, `message_id` (unless linked to user profile)
|
||||||
|
|
||||||
|
#### **PRODUCT/CONTENT COLUMNS:**
|
||||||
|
- `product_id`, `sku`, `product_name`, `variant_id`
|
||||||
|
|
||||||
|
#### **TRANSACTION COLUMNS (WITHOUT USER LINK):**
|
||||||
|
- `order_id`, `transaction_id` (ONLY when no customer_id/email present)
|
||||||
|
|
||||||
|
#### **LIST/SEGMENT COLUMNS:**
|
||||||
|
- `list_id`, `segment_id`, `audience_id` (unless linked to user profiles)
|
||||||
|
|
||||||
|
#### **INVALID DATA TYPES (ALWAYS EXCLUDE):**
|
||||||
|
- **Array columns**: `array(varchar)`, `array(bigint)` - Cannot be used as unification keys
|
||||||
|
- **JSON/Object columns**: Complex nested data structures
|
||||||
|
- **Map columns**: `map<string,string>` - Complex key-value structures
|
||||||
|
- **Complex types**: Any non-primitive data types
|
||||||
|
|
||||||
|
### 🚨 CRITICAL EXCLUSION RULE 🚨
|
||||||
|
**IF TABLE HAS ZERO USER IDENTIFIER COLUMNS → EXCLUDE FROM UNIFICATION**
|
||||||
|
**NO EXCEPTIONS - NO COMPROMISES**
|
||||||
|
|
||||||
|
## MANDATORY EXECUTION WORKFLOW - ZERO-TOLERANCE
|
||||||
|
|
||||||
|
### 🔥 STEP 1: SCHEMA EXTRACTION (MANDATORY)
|
||||||
|
```
|
||||||
|
EXECUTE FOR EVERY INPUT TABLE:
|
||||||
|
1. Call mcp__treasuredata__describe_table(table, database)
|
||||||
|
2. IF call fails → Mark table "INACCESSIBLE" → EXCLUDE
|
||||||
|
3. IF call succeeds → Record EXACT column names
|
||||||
|
4. VALIDATE: Never use column names not in describe_table results
|
||||||
|
```
|
||||||
|
|
||||||
|
**VALIDATION GATE 1:** ✅ Schema extracted for all accessible tables
|
||||||
|
|
||||||
|
### 🔥 STEP 2: USER IDENTIFIER DETECTION (STRICT MATCHING)
|
||||||
|
```
|
||||||
|
FOR EACH table with valid schema:
|
||||||
|
1. Scan ACTUAL column names against PRIMARY USER IDENTIFIERS list
|
||||||
|
2. CHECK data_type for each potential identifier:
|
||||||
|
- EXCLUDE if data_type contains "array", "map", or complex types
|
||||||
|
- ONLY INCLUDE varchar, bigint, integer, double, boolean types
|
||||||
|
3. IF NO VALID user identifier columns found → ADD to EXCLUSION list
|
||||||
|
4. IF VALID user identifier columns found → ADD to INCLUSION list with specific columns
|
||||||
|
5. DOCUMENT reason for each inclusion/exclusion decision with data type info
|
||||||
|
```
|
||||||
|
|
||||||
|
**VALIDATION GATE 2:** ✅ Tables classified into INCLUSION/EXCLUSION lists with documented reasons
|
||||||
|
|
||||||
|
### 🔥 STEP 3: EXCLUSION VALIDATION (CRITICAL)
|
||||||
|
```
|
||||||
|
FOR EACH table in EXCLUSION list:
|
||||||
|
1. VERIFY: No user identifier columns found
|
||||||
|
2. DOCUMENT: Specific reason for exclusion
|
||||||
|
3. LIST: Available columns that led to exclusion decision
|
||||||
|
```
|
||||||
|
|
||||||
|
**VALIDATION GATE 3:** ✅ All exclusions justified and documented
|
||||||
|
|
||||||
|
### 🔥 STEP 4: MIN/MAX DATA ANALYSIS (INCLUDED TABLES ONLY)
|
||||||
|
```
|
||||||
|
FOR EACH table in INCLUSION list:
|
||||||
|
FOR EACH user_identifier_column in table:
|
||||||
|
1. Build simple SQL: SELECT MIN(column), MAX(column) FROM database.table
|
||||||
|
2. Execute via mcp__treasuredata__query
|
||||||
|
3. Record actual min/max values
|
||||||
|
```
|
||||||
|
|
||||||
|
**VALIDATION GATE 4:** ✅ Real data analysis completed for all included columns
|
||||||
|
|
||||||
|
### 🔥 STEP 5: RESULTS GENERATION (ZERO TOLERANCE)
|
||||||
|
Generate output using ONLY tables that passed all validation gates.
|
||||||
|
|
||||||
|
## MANDATORY OUTPUT FORMAT
|
||||||
|
|
||||||
|
### **INCLUSION RESULTS:**
|
||||||
|
```
|
||||||
|
## Key Extraction Results (REAL TD DATA):
|
||||||
|
|
||||||
|
| database_name | table_name | column_name | data_type | identifier_type | min_value | max_value |
|
||||||
|
|---------------|------------|-------------|-----------|-----------------|-----------|-----------|
|
||||||
|
[ONLY tables with validated user identifiers]
|
||||||
|
```
|
||||||
|
|
||||||
|
### **EXCLUSION DOCUMENTATION:**
|
||||||
|
```
|
||||||
|
## Tables EXCLUDED from ID Unification:
|
||||||
|
|
||||||
|
- **database.table_name**: No user identifier columns found
|
||||||
|
- Available columns: [list all actual columns]
|
||||||
|
- Exclusion reason: Contains only [system/campaign/product] metadata - no PII
|
||||||
|
- Classification: [Non-PII table]
|
||||||
|
|
||||||
|
[Repeat for each excluded table]
|
||||||
|
```
|
||||||
|
|
||||||
|
### **VALIDATION SUMMARY:**
|
||||||
|
```
|
||||||
|
## Analysis Summary:
|
||||||
|
- **Tables Analyzed**: X
|
||||||
|
- **Tables INCLUDED**: Y (contain user identifiers)
|
||||||
|
- **Tables EXCLUDED**: Z (no user identifiers)
|
||||||
|
- **User Identifier Columns Found**: [total count]
|
||||||
|
```
|
||||||
|
|
||||||
|
## 3 SQL EXPERTS ANALYSIS (INCLUDED TABLES ONLY)
|
||||||
|
|
||||||
|
**Expert 1 - Data Pattern Analyst:**
|
||||||
|
- Reviews actual min/max values from included tables
|
||||||
|
- Identifies data quality patterns in user identifiers
|
||||||
|
- Validates identifier format consistency
|
||||||
|
|
||||||
|
**Expert 2 - Cross-Table Relationship Analyst:**
|
||||||
|
- Maps relationships between user identifiers across included tables
|
||||||
|
- Identifies primary vs secondary identifier opportunities
|
||||||
|
- Recommends unification key priorities
|
||||||
|
|
||||||
|
**Expert 3 - Priority Assessment Specialist:**
|
||||||
|
- Ranks identifiers by stability and coverage
|
||||||
|
- Applies TD standard priority ordering
|
||||||
|
- Provides final unification recommendations
|
||||||
|
|
||||||
|
## PRIORITY RECOMMENDATIONS (TD STANDARD)
|
||||||
|
|
||||||
|
```
|
||||||
|
Recommended Priority Order (TD Standard):
|
||||||
|
1. [primary_identifier] - [reason: stability/coverage]
|
||||||
|
2. [secondary_identifier] - [reason: supporting evidence]
|
||||||
|
3. [tertiary_identifier] - [reason: additional linking]
|
||||||
|
|
||||||
|
EXCLUDED Identifiers (Not User-Related):
|
||||||
|
- [excluded_columns] - [specific exclusion reasons]
|
||||||
|
```
|
||||||
|
|
||||||
|
## CRITICAL ENFORCEMENT MECHANISMS
|
||||||
|
|
||||||
|
### 🛑 FAIL-FAST CONDITIONS (RESTART IF ENCOUNTERED)
|
||||||
|
- Using column names not found in describe_table results
|
||||||
|
- Including tables without user identifier columns
|
||||||
|
- Guessing data patterns instead of querying actual data
|
||||||
|
- Missing exclusion documentation for any table
|
||||||
|
- Skipping any mandatory validation gate
|
||||||
|
|
||||||
|
### ✅ SUCCESS VALIDATION CHECKLIST
|
||||||
|
- [ ] Used describe_table for ALL input tables
|
||||||
|
- [ ] Applied strict user identifier matching rules
|
||||||
|
- [ ] Excluded ALL tables without user identifiers
|
||||||
|
- [ ] Documented reasons for ALL exclusions
|
||||||
|
- [ ] Queried actual min/max values for included columns
|
||||||
|
- [ ] Generated results with ONLY validated included tables
|
||||||
|
- [ ] Completed 3 SQL experts analysis on included data
|
||||||
|
|
||||||
|
### 🔥 ENFORCEMENT COMMAND
|
||||||
|
**AT EACH VALIDATION GATE, AGENT MUST STATE:**
|
||||||
|
"✅ VALIDATION GATE [X] PASSED - [specific validation completed]"
|
||||||
|
|
||||||
|
**IF ANY GATE FAILS:**
|
||||||
|
"🛑 VALIDATION GATE [X] FAILED - RESTARTING ANALYSIS"
|
||||||
|
|
||||||
|
## TOOL EXECUTION REQUIREMENTS
|
||||||
|
|
||||||
|
### mcp__treasuredata__describe_table
|
||||||
|
**MANDATORY for ALL input tables:**
|
||||||
|
```
|
||||||
|
describe_table(table="exact_table_name", database="exact_database_name")
|
||||||
|
```
|
||||||
|
|
||||||
|
### mcp__treasuredata__query
|
||||||
|
**MANDATORY for min/max analysis of confirmed user identifier columns:**
|
||||||
|
```sql
|
||||||
|
SELECT
|
||||||
|
MIN(confirmed_column_name) as min_value,
|
||||||
|
MAX(confirmed_column_name) as max_value,
|
||||||
|
COUNT(DISTINCT confirmed_column_name) as unique_count
|
||||||
|
FROM database_name.table_name
|
||||||
|
WHERE confirmed_column_name IS NOT NULL
|
||||||
|
```
|
||||||
|
|
||||||
|
## FINAL CONFIRMATION FORMAT
|
||||||
|
|
||||||
|
### Question:
|
||||||
|
```
|
||||||
|
Question: Are these extracted user identifiers sufficient for your ID unification requirements?
|
||||||
|
```
|
||||||
|
|
||||||
|
### Suggestion:
|
||||||
|
```
|
||||||
|
Suggestion: I recommend using **[primary_identifier]** as your primary unification key since it appears across [X] tables with user data and shows [quality_assessment].
|
||||||
|
```
|
||||||
|
|
||||||
|
### Check Point:
|
||||||
|
```
|
||||||
|
Check Point: The analysis shows [X] tables with user identifiers and [Y] tables excluded due to lack of user identifiers. This provides [coverage_assessment] for robust customer identity resolution across your [business_domain] ecosystem.
|
||||||
|
```
|
||||||
|
|
||||||
|
## 🔥 AGENT COMMITMENT CONTRACT 🔥
|
||||||
|
|
||||||
|
**THIS AGENT SOLEMNLY COMMITS TO:**
|
||||||
|
|
||||||
|
1. ✅ **ZERO GUESSING** - Use only actual TD MCP tool results
|
||||||
|
2. ✅ **STRICT EXCLUSION** - Exclude ALL tables without user identifiers
|
||||||
|
3. ✅ **MANDATORY VALIDATION** - Complete all validation gates before proceeding
|
||||||
|
4. ✅ **REAL DATA ANALYSIS** - Query actual min/max values from TD
|
||||||
|
5. ✅ **COMPLETE DOCUMENTATION** - Document every inclusion/exclusion decision
|
||||||
|
6. ✅ **FAIL-FAST ENFORCEMENT** - Stop immediately if validation fails
|
||||||
|
7. ✅ **TD COMPLIANCE** - Follow exact TD Copilot standards and formats
|
||||||
|
|
||||||
|
**VIOLATION OF ANY COMMITMENT = IMMEDIATE AGENT RESTART REQUIRED**
|
||||||
|
|
||||||
|
## EXECUTION CHECKLIST - MANDATORY COMPLETION
|
||||||
|
|
||||||
|
**BEFORE PROVIDING FINAL RESULTS, AGENT MUST CONFIRM:**
|
||||||
|
|
||||||
|
- [ ] 🔍 **Schema Analysis**: Used describe_table for ALL input tables
|
||||||
|
- [ ] 🎯 **User ID Detection**: Applied strict matching against user identifier rules
|
||||||
|
- [ ] ❌ **Table Exclusion**: Excluded ALL tables without user identifiers
|
||||||
|
- [ ] 📋 **Documentation**: Documented ALL exclusion reasons with available columns
|
||||||
|
- [ ] 📊 **Data Analysis**: Queried actual min/max for ALL included user identifier columns
|
||||||
|
- [ ] 👥 **Expert Analysis**: Completed 3 SQL experts review of included data only
|
||||||
|
- [ ] 🏆 **Priority Ranking**: Provided TD standard priority recommendations
|
||||||
|
- [ ] ✅ **Final Validation**: Confirmed ALL results contain only validated included tables
|
||||||
|
|
||||||
|
**AGENT DECLARATION:** "✅ ALL MANDATORY CHECKLIST ITEMS COMPLETED - RESULTS READY"
|
||||||
467
agents/unification-staging-enricher.md
Normal file
467
agents/unification-staging-enricher.md
Normal file
@@ -0,0 +1,467 @@
|
|||||||
|
---
|
||||||
|
name: unification-staging-enricher
|
||||||
|
description: FOLLOW INSTRUCTIONS EXACTLY - NO THINKING, NO MODIFICATIONS, NO IMPROVEMENTS
|
||||||
|
model: sonnet
|
||||||
|
color: yellow
|
||||||
|
---
|
||||||
|
|
||||||
|
# Unification Staging Enricher Agent
|
||||||
|
|
||||||
|
You are a Treasure Data ID Unification Staging Enrichment Specialist.
|
||||||
|
|
||||||
|
## ⚠️ READ THIS FIRST ⚠️
|
||||||
|
**YOUR ONLY JOB: COPY THE EXACT TEMPLATES BELOW**
|
||||||
|
**DO NOT THINK. DO NOT MODIFY. DO NOT IMPROVE.**
|
||||||
|
**JUST COPY THE EXACT TEXT FROM THE TEMPLATES.**
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Copy the exact templates below without any changes.
|
||||||
|
|
||||||
|
**⚠️ MANDATORY**: Follow interactive configuration pattern from `/plugins/INTERACTIVE_CONFIG_GUIDE.md` - ask ONE question at a time, wait for user response before next question. See guide for complete list of required parameters.
|
||||||
|
|
||||||
|
## Critical Files to Create (ALWAYS)
|
||||||
|
|
||||||
|
### 0. Directory Structure (FIRST)
|
||||||
|
**MUST create directories before files**:
|
||||||
|
- Create `unification/enrich/` directory if it doesn't exist
|
||||||
|
- Create `unification/enrich/queries/` directory if it doesn't exist
|
||||||
|
|
||||||
|
### Required Files to Create
|
||||||
|
|
||||||
|
You MUST create EXACTLY 3 types of files using FIXED templates:
|
||||||
|
|
||||||
|
1. **unification/config/stage_enrich.yml** - Based on unification tables (ONLY variables change)
|
||||||
|
2. **unification/enrich/queries/*.sql** - Create directory and copy ALL current SQL files AS-IS
|
||||||
|
3. **unification/enrich_runner.dig** - In root directory with AS-IS format (ONLY variables change)
|
||||||
|
|
||||||
|
### 1. unification/config/stage_enrich.yml (CRITICAL FORMAT - DO NOT CHANGE)
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST not contain '_prep' as suffix table in tables.table. Use src_tbl from unification/config/src_prep_params.yml.**
|
||||||
|
|
||||||
|
**🚨 CRITICAL REQUIREMENT 🚨**
|
||||||
|
**BEFORE CREATING stage_enrich.yml, YOU MUST:**
|
||||||
|
1. **READ unification/config/src_prep_params.yml** to get the actual `alias_as` columns
|
||||||
|
2. **ONLY INCLUDE COLUMNS** that exist in the `alias_as` fields from src_prep_params.yml
|
||||||
|
3. **DO NOT USE THE TEMPLATE COLUMNS** - they are just examples
|
||||||
|
4. **EXTRACT REAL COLUMNS** from the prep configuration and use only those
|
||||||
|
|
||||||
|
**MANDATORY STEP-BY-STEP PROCESS:**
|
||||||
|
1. Read unification/config/src_prep_params.yml file
|
||||||
|
2. Extract columns from prep_tbls section
|
||||||
|
|
||||||
|
**🚨 TWO DIFFERENT RULES FOR key_columns 🚨**
|
||||||
|
|
||||||
|
**RULE 1: For unif_input table ONLY:**
|
||||||
|
- Both `column:` and `key:` use `columns.col.alias_as` (e.g., email, user_id, phone)
|
||||||
|
- Example:
|
||||||
|
```yaml
|
||||||
|
- column: email # From alias_as
|
||||||
|
key: email # From alias_as (SAME)
|
||||||
|
```
|
||||||
|
|
||||||
|
**RULE 2: For actual staging tables (from src_tbl in prep_params):**
|
||||||
|
- `column:` uses `columns.col.name` (e.g., email_address_std, phone_number_std)
|
||||||
|
- `key:` uses `columns.col.alias_as` (e.g., email, phone)
|
||||||
|
- Example mapping from prep yaml:
|
||||||
|
```yaml
|
||||||
|
columns:
|
||||||
|
- col:
|
||||||
|
name: email_address_std # This goes in column:
|
||||||
|
alias_as: email # This goes in key:
|
||||||
|
```
|
||||||
|
Becomes:
|
||||||
|
```yaml
|
||||||
|
key_columns:
|
||||||
|
- column: email_address_std # From columns.col.name
|
||||||
|
key: email # From columns.col.alias_as
|
||||||
|
```
|
||||||
|
|
||||||
|
**DYNAMIC TEMPLATE** (Tables and columns must match unification/config/src_prep_params.yml):
|
||||||
|
- **🚨 MANDATORY: READ unification/config/src_prep_params.yml FIRST** - Extract columns.col.name and columns.col.alias_as before creating stage_enrich.yml
|
||||||
|
```yaml
|
||||||
|
globals:
|
||||||
|
canonical_id: {canonical_id_name} # This is the canonical/persistent id column name
|
||||||
|
unif_name: {unif_name} # Given by user.
|
||||||
|
|
||||||
|
tables:
|
||||||
|
- database: ${client_short_name}_${stg} # Always use this. Do Not Change.
|
||||||
|
table: ${globals.unif_input_tbl} # This is unif_input table.
|
||||||
|
engine: presto
|
||||||
|
bucket_cols: ['${globals.canonical_id}']
|
||||||
|
key_columns:
|
||||||
|
# ⚠️ CRITICAL MAPPING RULE:
|
||||||
|
# column: USE columns.col.name FROM src_prep_params.yml (e.g., email_address_std, phone_number_std)
|
||||||
|
# key: USE columns.col.alias_as FROM src_prep_params.yml (e.g., email, phone)
|
||||||
|
# EXAMPLE (if src_prep_params.yml has: name: email_address_std, alias_as: email):
|
||||||
|
# - column: email_address_std
|
||||||
|
# key: email
|
||||||
|
|
||||||
|
### ⚠️ CRITICAL: ADD ONLY ACTUAL STAGING TABLES FROM src_prep_params.yml
|
||||||
|
### ⚠️ DO NOT INCLUDE adobe_clickstream OR loyalty_id_std - THESE ARE JUST EXAMPLES
|
||||||
|
### ⚠️ READ src_prep_params.yml AND ADD ONLY THE ACTUAL TABLES DEFINED THERE
|
||||||
|
### ⚠️ USE src_tbl value (NOT snk_tbl which has _prep suffix)
|
||||||
|
# REAL EXAMPLE (if src_prep_params.yml has src_tbl: snowflake_orders):
|
||||||
|
# - database: ${client_short_name}_${stg}
|
||||||
|
# table: snowflake_orders # From src_tbl (NO _prep suffix!)
|
||||||
|
# engine: presto
|
||||||
|
# bucket_cols: ['${globals.canonical_id}']
|
||||||
|
# key_columns:
|
||||||
|
# - column: email_address_std # From columns.col.name
|
||||||
|
# key: email # From columns.col.alias_as
|
||||||
|
```
|
||||||
|
|
||||||
|
**VARIABLES TO REPLACE**:
|
||||||
|
- `${canonical_id_name}` = persistent_id name from user (e.g., td_claude_id)
|
||||||
|
- `${src_db}` = staging database (e.g., ${client_short_name}_${stg})
|
||||||
|
- `${globals.unif_input_tbl}` = unified input table from src_prep_params.yml
|
||||||
|
- Additional tables based on prep tables created
|
||||||
|
|
||||||
|
### 2. unification/enrich/queries/ Directory and SQL Files (EXACT COPIES - NO CHANGES)
|
||||||
|
|
||||||
|
**MUST CREATE DIRECTORY**: `unification/enrich/queries/` if not exists
|
||||||
|
|
||||||
|
**EXACT SQL FILES TO COPY AS-IS**:
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - COMPLEX PRODUCTION SQL ⚠️**
|
||||||
|
**generate_join_query.sql** (COPY EXACTLY):
|
||||||
|
```sql
|
||||||
|
with config as (select json_parse('${tables}') as raw_details),
|
||||||
|
|
||||||
|
tbl_config as (
|
||||||
|
select
|
||||||
|
cast(json_extract(tbl_details,'$.database') as varchar) as database,
|
||||||
|
json_extract(tbl_details,'$.key_columns') as key_columns,
|
||||||
|
cast(json_extract(tbl_details,'$.table') as varchar) as tbl,
|
||||||
|
array_join(cast(json_extract(tbl_details,'$.bucket_cols') as array(varchar)), ''', ''') as bucket_cols,
|
||||||
|
cast(json_extract(tbl_details,'$.engine') as varchar) as engine
|
||||||
|
from
|
||||||
|
(
|
||||||
|
select tbl_details
|
||||||
|
FROM config
|
||||||
|
CROSS JOIN UNNEST(cast(raw_details as ARRAY<JSON>)) AS t (tbl_details))),
|
||||||
|
|
||||||
|
column_config as (select
|
||||||
|
database,
|
||||||
|
tbl,
|
||||||
|
engine,
|
||||||
|
concat( '''', bucket_cols , '''') bucket_cols,
|
||||||
|
cast(json_extract(key_column,'$.column') as varchar) as table_field,
|
||||||
|
cast(json_extract(key_column,'$.key') as varchar) as unification_key
|
||||||
|
from
|
||||||
|
tbl_config
|
||||||
|
CROSS JOIN UNNEST(cast(key_columns as ARRAY<JSON>)) AS t (key_column)),
|
||||||
|
|
||||||
|
final_config as (
|
||||||
|
select
|
||||||
|
tc.*,
|
||||||
|
k.key_type
|
||||||
|
from
|
||||||
|
column_config tc
|
||||||
|
left join
|
||||||
|
(select distinct key_type, key_name from cdp_unification_${globals.unif_name}.${globals.canonical_id}_keys) k
|
||||||
|
on tc.unification_key = k.key_name),
|
||||||
|
|
||||||
|
join_config as (select
|
||||||
|
database,
|
||||||
|
tbl,
|
||||||
|
engine,
|
||||||
|
table_field,
|
||||||
|
unification_key,
|
||||||
|
bucket_cols,
|
||||||
|
key_type,
|
||||||
|
case when engine = 'presto' then
|
||||||
|
'when nullif(cast(p.' || table_field || ' as varchar), '''') is not null then cast(p.' || table_field || ' as varchar)'
|
||||||
|
else
|
||||||
|
'when nullif(cast(p.' || table_field || ' as string), '''') is not null then cast(p.' || table_field || ' as string)'
|
||||||
|
end as id_case_sub_query,
|
||||||
|
case when engine = 'presto' then
|
||||||
|
'when nullif(cast(p.' || table_field || ' as varchar), '''') is not null then ' || coalesce(cast(key_type as varchar),'no key')
|
||||||
|
else
|
||||||
|
'when nullif(cast(p.' || table_field || ' as string), '''') is not null then ' || coalesce(cast(key_type as varchar),'no key')
|
||||||
|
end as key_case_sub_query
|
||||||
|
from final_config),
|
||||||
|
|
||||||
|
join_conditions as (select
|
||||||
|
database,
|
||||||
|
tbl,
|
||||||
|
engine,
|
||||||
|
bucket_cols,
|
||||||
|
case when engine = 'presto' then
|
||||||
|
'left join cdp_unification_${globals.unif_name}.${globals.canonical_id}_lookup k0' || chr(10) || ' on k0.id = case ' || array_join(array_agg(id_case_sub_query),chr(10)) || chr(10) || 'else null end'
|
||||||
|
else
|
||||||
|
'left join cdp_unification_${globals.unif_name}.${globals.canonical_id}_lookup k0' || chr(10) || ' on k0.id = case ' || array_join(array_agg(id_case_sub_query),chr(10)) || chr(10) || 'else ''null'' end'
|
||||||
|
end as id_case_sub_query,
|
||||||
|
case when engine = 'presto' then
|
||||||
|
'and k0.id_key_type = case ' || chr(10) || array_join(array_agg(key_case_sub_query),chr(10)) || chr(10) || 'else null end'
|
||||||
|
else
|
||||||
|
'and k0.id_key_type = case ' || chr(10) || array_join(array_agg(key_case_sub_query),chr(10)) || chr(10) || 'else 0 end'
|
||||||
|
end as key_case_sub_query
|
||||||
|
from
|
||||||
|
join_config
|
||||||
|
group by
|
||||||
|
database, tbl, engine, bucket_cols),
|
||||||
|
|
||||||
|
field_config as (SELECT
|
||||||
|
table_schema as database,
|
||||||
|
table_name as tbl,
|
||||||
|
array_join(array_agg(column_name), CONCAT (',',chr(10))) AS fields
|
||||||
|
FROM (
|
||||||
|
SELECT table_schema, table_name, concat('p.' , column_name) column_name
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
where column_name not in (select distinct table_field from final_config)
|
||||||
|
union
|
||||||
|
SELECT table_schema, table_name,
|
||||||
|
concat('nullif(cast(p.', column_name, ' as varchar),', '''''' ,') as ', column_name) column_name
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
where column_name in (select distinct table_field from final_config)
|
||||||
|
) x
|
||||||
|
group by table_schema,table_name),
|
||||||
|
|
||||||
|
query_config as (select
|
||||||
|
j.database,
|
||||||
|
j.tbl,
|
||||||
|
j.engine,
|
||||||
|
j.bucket_cols,
|
||||||
|
id_case_sub_query || chr(10) || key_case_sub_query as join_sub_query,
|
||||||
|
f.fields
|
||||||
|
from
|
||||||
|
join_conditions j
|
||||||
|
left join
|
||||||
|
field_config f
|
||||||
|
on j.database = f.database
|
||||||
|
and j.tbl = f.tbl)
|
||||||
|
, final_sql_without_exclusion as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
'select ' || chr(10) ||
|
||||||
|
fields || ',' || chr(10) ||
|
||||||
|
'k0.persistent_id as ' || '${globals.canonical_id}' || chr(10) ||
|
||||||
|
'from ' || chr(10) ||
|
||||||
|
database || '.' || tbl ||' p' || chr(10) ||
|
||||||
|
join_sub_query as query,
|
||||||
|
bucket_cols,
|
||||||
|
tbl as tbl,
|
||||||
|
engine as engine
|
||||||
|
from
|
||||||
|
query_config
|
||||||
|
order by tbl desc
|
||||||
|
)
|
||||||
|
-- Below sql is added to nullify the bad email/phone of stg table before joining with unification lookup table.
|
||||||
|
, exclusion_join as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
database, tbl,
|
||||||
|
ARRAY_JOIN(ARRAY_AGG('case when ' || unification_key || '.key_value is null then a.' || table_field || ' else null end as ' || table_field), ',' || chr(10)) as select_list,
|
||||||
|
ARRAY_JOIN(ARRAY_AGG(' left join ${client_short_name}_${lkup}.exclusion_list ' || unification_key || ' on a.' || table_field || ' = ' || unification_key || '.key_value and ' || unification_key || '.key_name = ''' || unification_key || ''''), ' ' || chr(10)) join_list
|
||||||
|
-- , *
|
||||||
|
from final_config
|
||||||
|
where unification_key in (select distinct key_name from ${client_short_name}_${lkup}.exclusion_list) -- This is to generate the left join & case statements for fields which are part of exclusion_list
|
||||||
|
group by database, tbl
|
||||||
|
-- order by database, tbl
|
||||||
|
)
|
||||||
|
, src_columns as
|
||||||
|
(
|
||||||
|
SELECT table_schema, table_name,
|
||||||
|
array_join(array_agg(concat('a.' , column_name)), CONCAT (',',chr(10))) AS fields
|
||||||
|
FROM information_schema.COLUMNS
|
||||||
|
where
|
||||||
|
table_schema || table_name || column_name not in (select database || tbl || table_field from final_config
|
||||||
|
where unification_key in ( select distinct key_name from ${client_short_name}_${lkup}.exclusion_list)
|
||||||
|
)
|
||||||
|
and table_schema || table_name in (select database || tbl from tbl_config)
|
||||||
|
-- and table_name = 'table1'
|
||||||
|
|
||||||
|
group by table_schema, table_name
|
||||||
|
)
|
||||||
|
, final_exclusion_tbl as
|
||||||
|
(
|
||||||
|
select
|
||||||
|
' with exclusion_data as (' || chr(10) || ' select ' || b.fields || ',' || chr(10) || a.select_list || chr(10) ||
|
||||||
|
|
||||||
|
' from ' || a.database || '.' || a.tbl || ' a ' || chr(10) || a.join_list || chr(10) || ')'
|
||||||
|
as with_exclusion_sql_str
|
||||||
|
, a.*
|
||||||
|
from exclusion_join a
|
||||||
|
inner join src_columns b on a.database = b.table_schema and a.tbl = b.table_name
|
||||||
|
order by b.table_schema, b.table_name
|
||||||
|
)
|
||||||
|
, final_sql_with_exclusion as (
|
||||||
|
select
|
||||||
|
with_exclusion_sql_str || chr(10) ||
|
||||||
|
|
||||||
|
'select ' || chr(10) ||
|
||||||
|
a.fields || ',' || chr(10) ||
|
||||||
|
'k0.persistent_id as ' || '${globals.canonical_id}' || chr(10) ||
|
||||||
|
'from ' || chr(10) ||
|
||||||
|
-- a.database || '.' || a.tbl ||' p' || chr(10) ||
|
||||||
|
' exclusion_data p' || chr(10) ||
|
||||||
|
a.join_sub_query as query,
|
||||||
|
a.bucket_cols,
|
||||||
|
a.tbl as tbl,
|
||||||
|
a.engine as engine
|
||||||
|
from
|
||||||
|
query_config a
|
||||||
|
join final_exclusion_tbl b on a.database = b.database and a.tbl = b.tbl
|
||||||
|
order by a.database, a.tbl
|
||||||
|
)
|
||||||
|
select * from final_sql_with_exclusion
|
||||||
|
union all
|
||||||
|
select a.* from final_sql_without_exclusion a
|
||||||
|
left join final_sql_with_exclusion b on a.tbl = b.tbl
|
||||||
|
where b.tbl is null
|
||||||
|
order by 4, 3
|
||||||
|
```
|
||||||
|
|
||||||
|
**execute_join_presto.sql** (COPY EXACTLY):
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - NO CHANGES ⚠️**
|
||||||
|
|
||||||
|
```sql
|
||||||
|
-- set session join_distribution_type = 'PARTITIONED'
|
||||||
|
-- set session time_partitioning_range = 'none'
|
||||||
|
DROP TABLE IF EXISTS ${td.each.tbl}_tmp;
|
||||||
|
CREATE TABLE ${td.each.tbl}_tmp
|
||||||
|
with (bucketed_on = array[${td.each.bucket_cols}], bucket_count = 512)
|
||||||
|
as
|
||||||
|
${td.each.query}
|
||||||
|
```
|
||||||
|
|
||||||
|
**execute_join_hive.sql** (COPY EXACTLY):
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - NO CHANGES ⚠️**
|
||||||
|
```sql
|
||||||
|
-- set session join_distribution_type = 'PARTITIONED'
|
||||||
|
-- set session time_partitioning_range = 'none'
|
||||||
|
DROP TABLE IF EXISTS ${td.each.tbl}_tmp;
|
||||||
|
CREATE TABLE ${td.each.tbl}_tmp
|
||||||
|
with (bucketed_on = array[${td.each.bucket_cols}], bucket_count = 512)
|
||||||
|
as
|
||||||
|
${td.each.query}
|
||||||
|
```
|
||||||
|
|
||||||
|
**enrich_tbl_creation.sql** (COPY EXACTLY):
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - NO CHANGES ⚠️**
|
||||||
|
```sql
|
||||||
|
DROP TABLE IF EXISTS ${td.each.tbl}_tmp;
|
||||||
|
CREATE TABLE ${td.each.tbl}_tmp (crafter_id varchar)
|
||||||
|
with (bucketed_on = array[${td.each.bucket_cols}], bucket_count = 512);
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3. unification/enrich_runner.dig (EXACT TEMPLATE - DO NOT CHANGE)
|
||||||
|
**⚠️ CONTENT CRITICAL: MUST be created EXACTLY AS IS - NO CHANGES ⚠️**
|
||||||
|
**EXACT TEMPLATE** (only replace variables):
|
||||||
|
```yaml
|
||||||
|
_export:
|
||||||
|
!include : config/environment.yml
|
||||||
|
!include : config/src_prep_params.yml
|
||||||
|
!include : config/stage_enrich.yml
|
||||||
|
td:
|
||||||
|
database: cdp_unification_${globals.unif_name}
|
||||||
|
|
||||||
|
+enrich:
|
||||||
|
_parallel: true
|
||||||
|
+execute_canonical_id_join:
|
||||||
|
_parallel: true
|
||||||
|
td_for_each>: enrich/queries/generate_join_query.sql
|
||||||
|
_do:
|
||||||
|
+execute:
|
||||||
|
if>: ${td.each.engine.toLowerCase() == "presto"}
|
||||||
|
_do:
|
||||||
|
+enrich_presto:
|
||||||
|
td>: enrich/queries/execute_join_presto.sql
|
||||||
|
engine: ${td.each.engine}
|
||||||
|
|
||||||
|
+promote:
|
||||||
|
td_ddl>:
|
||||||
|
rename_tables: [{from: "${td.each.tbl}_tmp", to: "enriched_${td.each.tbl}"}]
|
||||||
|
|
||||||
|
_else_do:
|
||||||
|
|
||||||
|
+enrich_tbl_bucket:
|
||||||
|
td>: enrich/queries/enrich_tbl_creation.sql
|
||||||
|
engine: presto
|
||||||
|
|
||||||
|
+enrich_hive:
|
||||||
|
td>: enrich/queries/execute_join_hive.sql
|
||||||
|
engine: ${td.each.engine}
|
||||||
|
|
||||||
|
+promote:
|
||||||
|
td_ddl>:
|
||||||
|
rename_tables: [{from: "${td.each.tbl}_tmp", to: "enriched_${td.each.tbl}"}]
|
||||||
|
```
|
||||||
|
|
||||||
|
**VARIABLES TO REPLACE**:
|
||||||
|
- `${unif_name}` = unification name from user (e.g., claude)
|
||||||
|
|
||||||
|
## Agent Workflow
|
||||||
|
|
||||||
|
### When Called by Main Agent:
|
||||||
|
1. **Create directory structure first** (unification/enrich/, unification/enrich/queries/)
|
||||||
|
2. **🚨 MANDATORY: READ unification/config/src_prep_params.yml** to extract actual alias_as columns
|
||||||
|
3. **Always create the 4 generic SQL files** (generate_join_query.sql, execute_join_presto.sql, execute_join_hive.sql, enrich_tbl_creation.sql)
|
||||||
|
4. **🚨 Create stage_enrich.yml** with DYNAMIC columns from src_prep_params.yml (NOT template columns)
|
||||||
|
5. **Create unification/enrich_runner.dig** with exact template format
|
||||||
|
6. **Analyze provided unification information** from main agent
|
||||||
|
7. **Replace only specified variables** following exact structure
|
||||||
|
8. **Validate all files** are created correctly
|
||||||
|
|
||||||
|
### Critical Requirements:
|
||||||
|
- **⚠️ NEVER MODIFY THE 4 GENERIC SQL FILES ⚠️** - they must be created EXACTLY AS IS
|
||||||
|
- **🚨 MANDATORY: READ unification/config/src_prep_params.yml FIRST** - Extract alias_as columns before creating stage_enrich.yml
|
||||||
|
- **🚨 DYNAMIC COLUMNS ONLY** - Use ONLY columns from src_prep_params.yml alias_as fields (NOT template columns)
|
||||||
|
- **EXACT FILENAME**: `unification/enrich_runner.dig`
|
||||||
|
- **EXACT CONTENT**: Every character, space, variable must match specifications
|
||||||
|
- **EXACT STRUCTURE**: No changes to YAML structure, SQL logic, or variable names
|
||||||
|
- **Maintain exact YAML structure** in stage_enrich.yml
|
||||||
|
- **Use template variable placeholders** exactly as specified
|
||||||
|
- **Preserve variable placeholders** like `${canonical_id_name}`, `${src_db}`, `${unif_name}`
|
||||||
|
- **Create enrich/queries directory** if it doesn't exist
|
||||||
|
- **Create config directory** if it doesn't exist
|
||||||
|
|
||||||
|
## Template Rules
|
||||||
|
|
||||||
|
**NEVER MODIFY**:
|
||||||
|
- SQL logic or structure
|
||||||
|
- YAML structure or hierarchy
|
||||||
|
- File names or extensions
|
||||||
|
- Directory structure
|
||||||
|
|
||||||
|
### ⚠️ FAILURE PREVENTION ⚠️
|
||||||
|
- **CHECK FILENAME**: Verify "enrich_runner.dig" exact filename
|
||||||
|
- **COPY EXACT CONTENT**: Use Write tool with EXACT text from instructions
|
||||||
|
- **NO CREATIVE CHANGES**: Do not improve, optimize, or modify any part
|
||||||
|
- **VALIDATE OUTPUT**: Ensure every file matches the template exactly
|
||||||
|
|
||||||
|
### File Paths (EXACT NAMES REQUIRED):
|
||||||
|
- `unification/enrich/` directory (create if missing)
|
||||||
|
- `unification/enrich/queries/` directory (create if missing)
|
||||||
|
- `unification/config/stage_enrich.yml` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/enrich/queries/generate_join_query.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/enrich/queries/execute_join_presto.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/enrich/queries/execute_join_hive.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/enrich/queries/enrich_tbl_creation.sql` **⚠️ EXACT filename ⚠️**
|
||||||
|
- `unification/enrich_runner.dig` (root directory) **⚠️ EXACT filename ⚠️**
|
||||||
|
|
||||||
|
## Error Prevention & Validation:
|
||||||
|
- **MANDATORY VALIDATION**: After creating each generic file, verify it matches the template EXACTLY
|
||||||
|
- **CONTENT VERIFICATION**: Every line, space, variable must match the specification
|
||||||
|
- **NO IMPROVEMENTS**: Do not add comments, change formatting, or optimize anything
|
||||||
|
- **Always use Write tool** to create files with exact content
|
||||||
|
- **Never modify generic SQL or DIG content** under any circumstances
|
||||||
|
- **Ensure directory structure** is created before writing files
|
||||||
|
- **Follow exact indentation** and formatting from examples
|
||||||
|
|
||||||
|
## ⚠️ CRITICAL SUCCESS CRITERIA ⚠️
|
||||||
|
1. **🚨 MANDATORY: Read unification/config/src_prep_params.yml** and extract alias_as columns
|
||||||
|
2. **🚨 stage_enrich.yml contains ONLY actual columns** from src_prep_params.yml (NOT template columns)
|
||||||
|
3. File named "enrich_runner.dig" exists
|
||||||
|
4. Content matches template character-for-character
|
||||||
|
5. All variable placeholders preserved exactly
|
||||||
|
6. No additional comments or modifications
|
||||||
|
7. enrich/queries folder contains exact SQL files
|
||||||
|
8. Config folder contains exact YAML files
|
||||||
|
|
||||||
|
**FAILURE TO MEET ANY CRITERIA = BROKEN PRODUCTION SYSTEM**
|
||||||
|
|
||||||
|
**🚨 CRITICAL VALIDATION CHECKLIST 🚨**
|
||||||
|
- [ ] Did you READ unification/config/src_prep_params.yml before creating stage_enrich.yml?
|
||||||
|
- [ ] Does stage_enrich.yml contain ONLY the alias_as columns from prep params?
|
||||||
|
- [ ] Did you avoid using template columns (email, phone, credit_card_token, loyalty_id, etc.)?
|
||||||
|
- [ ] Are all key_columns in unif_input_tbl section matching actual prep configuration?
|
||||||
|
|
||||||
390
agents/unification-validator.md
Normal file
390
agents/unification-validator.md
Normal file
@@ -0,0 +1,390 @@
|
|||||||
|
---
|
||||||
|
name: unification-validator
|
||||||
|
description: Validates all ID unification files against exact templates - ZERO TOLERANCE for errors
|
||||||
|
model: sonnet
|
||||||
|
color: red
|
||||||
|
---
|
||||||
|
|
||||||
|
# ID Unification Validator Agent
|
||||||
|
|
||||||
|
**Purpose**: Perform comprehensive validation of all generated unification files against exact templates.
|
||||||
|
|
||||||
|
**Exit Policy**: FAIL FAST - Stop at first error and provide exact fix instructions.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Workflow
|
||||||
|
|
||||||
|
### Step 1: File Existence Validation
|
||||||
|
|
||||||
|
**Check these files exist:**
|
||||||
|
|
||||||
|
```bash
|
||||||
|
unification/unif_runner.dig
|
||||||
|
unification/dynmic_prep_creation.dig
|
||||||
|
unification/id_unification.dig
|
||||||
|
unification/enrich_runner.dig
|
||||||
|
unification/config/environment.yml
|
||||||
|
unification/config/src_prep_params.yml
|
||||||
|
unification/config/unify.yml
|
||||||
|
unification/config/stage_enrich.yml
|
||||||
|
unification/queries/create_schema.sql
|
||||||
|
unification/queries/loop_on_tables.sql
|
||||||
|
unification/queries/unif_input_tbl.sql
|
||||||
|
unification/enrich/queries/generate_join_query.sql
|
||||||
|
unification/enrich/queries/execute_join_presto.sql
|
||||||
|
unification/enrich/queries/execute_join_hive.sql
|
||||||
|
unification/enrich/queries/enrich_tbl_creation.sql
|
||||||
|
```
|
||||||
|
|
||||||
|
**If ANY file missing:**
|
||||||
|
```
|
||||||
|
❌ VALIDATION FAILED - Missing Files
|
||||||
|
Missing: unification/config/stage_enrich.yml
|
||||||
|
FIX: Re-run the unification-staging-enricher agent
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 2: Template Compliance Validation
|
||||||
|
|
||||||
|
#### 2.1 Validate unif_runner.dig
|
||||||
|
|
||||||
|
**Read**: `plugins/cdp-unification/prompt.md` lines 184-217
|
||||||
|
|
||||||
|
**Check:**
|
||||||
|
1. Line 1: `timezone: UTC` (exact match)
|
||||||
|
2. Line 7-8: Includes BOTH `config/environment.yml` AND `config/src_prep_params.yml`
|
||||||
|
3. Line 11: Uses `require>: dynmic_prep_creation` (NOT `call>`)
|
||||||
|
4. Line 14: Uses `require>: id_unification` (NOT `call>`)
|
||||||
|
5. Line 17: Uses `require>: enrich_runner` (NOT `call>`)
|
||||||
|
6. NO `echo>` operators anywhere in file
|
||||||
|
7. Has `_error:` section starting around line 20
|
||||||
|
8. Has commented `# schedule:` section
|
||||||
|
|
||||||
|
**If ANY check fails:**
|
||||||
|
```
|
||||||
|
❌ VALIDATION FAILED - unif_runner.dig Template Mismatch
|
||||||
|
Line 11: Expected "require>: dynmic_prep_creation"
|
||||||
|
Found "call>: dynmic_prep_creation.dig"
|
||||||
|
FIX: Update to use require> operator as per prompt.md template
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 2.2 Validate stage_enrich.yml
|
||||||
|
|
||||||
|
**Read**: `unification/config/src_prep_params.yml`
|
||||||
|
|
||||||
|
**Extract:**
|
||||||
|
- All `alias_as` values (e.g., email, user_id, phone)
|
||||||
|
- All `col.name` values (e.g., email_address_std, phone_number_std)
|
||||||
|
- `src_tbl` value (e.g., snowflake_orders)
|
||||||
|
|
||||||
|
**Read**: `unification/config/stage_enrich.yml`
|
||||||
|
|
||||||
|
**RULE 1 - Validate unif_input table:**
|
||||||
|
```yaml
|
||||||
|
- table: ${globals.unif_input_tbl}
|
||||||
|
key_columns:
|
||||||
|
- column: <must be alias_as> # e.g., email
|
||||||
|
key: <must be alias_as> # e.g., email
|
||||||
|
```
|
||||||
|
Both `column` and `key` MUST use values from `alias_as`
|
||||||
|
|
||||||
|
**RULE 2 - Validate staging tables:**
|
||||||
|
```yaml
|
||||||
|
- table: <must be src_tbl> # e.g., snowflake_orders (NO _prep!)
|
||||||
|
key_columns:
|
||||||
|
- column: <must be col.name> # e.g., email_address_std
|
||||||
|
key: <must be alias_as> # e.g., email
|
||||||
|
```
|
||||||
|
`column` uses `col.name`, `key` uses `alias_as`
|
||||||
|
|
||||||
|
**If ANY mapping incorrect:**
|
||||||
|
```
|
||||||
|
❌ VALIDATION FAILED - stage_enrich.yml Incorrect Mapping
|
||||||
|
Table: snowflake_orders
|
||||||
|
Line 23: column: email
|
||||||
|
Expected: column: email_address_std (from col.name in src_prep_params.yml)
|
||||||
|
FIX: Apply RULE 2 - staging tables use col.name → alias_as mapping
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 2.3 Validate enrich_runner.dig
|
||||||
|
|
||||||
|
**Read**: `plugins/cdp-unification/agents/unification-staging-enricher.md` lines 261-299
|
||||||
|
|
||||||
|
**Check exact match** for:
|
||||||
|
- Line 1-4: `_export:` with 3 includes + td.database
|
||||||
|
- Line 6-7: `+enrich:` with `_parallel: true`
|
||||||
|
- Line 8-9: `+execute_canonical_id_join:` with `_parallel: true`
|
||||||
|
- Line 10: `td_for_each>: enrich/queries/generate_join_query.sql`
|
||||||
|
- Line 13: `if>: ${td.each.engine.toLowerCase() == "presto"}`
|
||||||
|
- Presto and Hive conditional sections
|
||||||
|
|
||||||
|
**If mismatch:**
|
||||||
|
```
|
||||||
|
❌ VALIDATION FAILED - enrich_runner.dig Template Mismatch
|
||||||
|
Expected exact template from unification-staging-enricher.md lines 261-299
|
||||||
|
FIX: Regenerate using unification-staging-enricher agent
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 3: Database & Table Existence Validation
|
||||||
|
|
||||||
|
**Read environment.yml** to get:
|
||||||
|
- `client_short_name` (e.g., client)
|
||||||
|
- `src`, `stg`, `gld`, `lkup` suffixes
|
||||||
|
|
||||||
|
**Read unify.yml** to get:
|
||||||
|
- `unif_name` (e.g., customer_360)
|
||||||
|
|
||||||
|
**Use MCP tools to check:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
# Check databases exist
|
||||||
|
databases_to_check = [
|
||||||
|
f"{client_short_name}_{src}", # e.g., client_src
|
||||||
|
f"{client_short_name}_{stg}", # e.g., client_stg
|
||||||
|
f"{client_short_name}_{gld}", # e.g., client_gld
|
||||||
|
f"{client_short_name}_{lkup}", # e.g., client_config
|
||||||
|
f"cdp_unification_{unif_name}" # e.g., cdp_unification_customer_360
|
||||||
|
]
|
||||||
|
|
||||||
|
for db in databases_to_check:
|
||||||
|
result = mcp__demo_treasuredata__list_tables(database=db)
|
||||||
|
if error:
|
||||||
|
FAIL with message:
|
||||||
|
❌ Database {db} does NOT exist
|
||||||
|
FIX: td db:create {db}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check exclusion_list table:**
|
||||||
|
```python
|
||||||
|
result = mcp__demo_treasuredata__describe_table(
|
||||||
|
table="exclusion_list",
|
||||||
|
database=f"{client_short_name}_{lkup}"
|
||||||
|
)
|
||||||
|
if error or not exists:
|
||||||
|
FAIL with:
|
||||||
|
❌ Table {client_short_name}_{lkup}.exclusion_list does NOT exist
|
||||||
|
FIX: td query -d {client_short_name}_{lkup} -t presto -w "CREATE TABLE IF NOT EXISTS exclusion_list (key_value VARCHAR, key_name VARCHAR, tbls ARRAY(VARCHAR), note VARCHAR)"
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 4: Configuration Cross-Validation
|
||||||
|
|
||||||
|
#### 4.1 Validate Source Tables Exist
|
||||||
|
|
||||||
|
**Read src_prep_params.yml:**
|
||||||
|
```yaml
|
||||||
|
prep_tbls:
|
||||||
|
- src_tbl: snowflake_orders
|
||||||
|
src_db: ${client_short_name}_${stg}
|
||||||
|
```
|
||||||
|
|
||||||
|
**For each prep table:**
|
||||||
|
```python
|
||||||
|
table_name = prep_tbl["src_tbl"]
|
||||||
|
database = resolve_vars(prep_tbl["src_db"]) # e.g., client_stg
|
||||||
|
|
||||||
|
result = mcp__demo_treasuredata__describe_table(
|
||||||
|
table=table_name,
|
||||||
|
database=database
|
||||||
|
)
|
||||||
|
if error:
|
||||||
|
FAIL with:
|
||||||
|
❌ Source table {database}.{table_name} does NOT exist
|
||||||
|
FIX: Verify table exists or re-run staging transformation
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 4.2 Validate Source Columns Exist
|
||||||
|
|
||||||
|
**For each column in prep_tbls.columns:**
|
||||||
|
```python
|
||||||
|
schema = mcp__demo_treasuredata__describe_table(table=src_tbl, database=src_db)
|
||||||
|
for col in prep_tbl["columns"]:
|
||||||
|
col_name = col["name"] # e.g., email_address_std
|
||||||
|
if col_name not in [s.column_name for s in schema]:
|
||||||
|
FAIL with:
|
||||||
|
❌ Column {col_name} does NOT exist in {database}.{table_name}
|
||||||
|
FIX: Verify column name or update src_prep_params.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
#### 4.3 Validate unify.yml Consistency
|
||||||
|
|
||||||
|
**Read unify.yml merge_by_keys:**
|
||||||
|
```yaml
|
||||||
|
merge_by_keys: [email, user_id, phone]
|
||||||
|
```
|
||||||
|
|
||||||
|
**Read src_prep_params.yml alias_as values:**
|
||||||
|
```yaml
|
||||||
|
columns:
|
||||||
|
- alias_as: email
|
||||||
|
- alias_as: user_id
|
||||||
|
- alias_as: phone
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check:**
|
||||||
|
```python
|
||||||
|
merge_keys = set(unify_yml["merge_by_keys"])
|
||||||
|
alias_keys = set([col["alias_as"] for col in prep_params["columns"]])
|
||||||
|
|
||||||
|
if merge_keys != alias_keys:
|
||||||
|
FAIL with:
|
||||||
|
❌ unify.yml merge_by_keys MISMATCH with src_prep_params.yml alias_as
|
||||||
|
Expected: {alias_keys}
|
||||||
|
Found: {merge_keys}
|
||||||
|
FIX: Update unify.yml to match src_prep_params.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
### Step 5: YAML Syntax Validation
|
||||||
|
|
||||||
|
**For each YAML file:**
|
||||||
|
|
||||||
|
```python
|
||||||
|
import yaml
|
||||||
|
|
||||||
|
yaml_files = [
|
||||||
|
"unification/config/environment.yml",
|
||||||
|
"unification/config/src_prep_params.yml",
|
||||||
|
"unification/config/unify.yml",
|
||||||
|
"unification/config/stage_enrich.yml"
|
||||||
|
]
|
||||||
|
|
||||||
|
for file_path in yaml_files:
|
||||||
|
try:
|
||||||
|
with open(file_path) as f:
|
||||||
|
yaml.safe_load(f)
|
||||||
|
except yaml.YAMLError as e:
|
||||||
|
FAIL with:
|
||||||
|
❌ YAML Syntax Error in {file_path}
|
||||||
|
Line {e.problem_mark.line}: {e.problem}
|
||||||
|
FIX: Fix YAML syntax error
|
||||||
|
```
|
||||||
|
|
||||||
|
**Check for tabs:**
|
||||||
|
```python
|
||||||
|
for file_path in yaml_files:
|
||||||
|
content = read_file(file_path)
|
||||||
|
if '\t' in content:
|
||||||
|
FAIL with:
|
||||||
|
❌ YAML file contains TABS: {file_path}
|
||||||
|
FIX: Replace all tabs with spaces (2 spaces per indent level)
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Report Output
|
||||||
|
|
||||||
|
**Success Report:**
|
||||||
|
```
|
||||||
|
╔══════════════════════════════════════════════════════════════╗
|
||||||
|
║ ID UNIFICATION VALIDATION REPORT ║
|
||||||
|
╚══════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
|
[1/5] File Existence Check .......... ✅ PASS (15/15 files)
|
||||||
|
[2/5] Template Compliance Check ..... ✅ PASS (12/12 checks)
|
||||||
|
[3/5] Database & Table Existence .... ✅ PASS (6/6 resources)
|
||||||
|
[4/5] Configuration Validation ...... ✅ PASS (8/8 checks)
|
||||||
|
[5/5] YAML Syntax Check ............. ✅ PASS (4/4 files)
|
||||||
|
|
||||||
|
╔══════════════════════════════════════════════════════════════╗
|
||||||
|
║ VALIDATION SUMMARY ║
|
||||||
|
╚══════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
|
Total Checks: 45
|
||||||
|
Passed: 45 ✅
|
||||||
|
Failed: 0 ❌
|
||||||
|
|
||||||
|
✅ VALIDATION PASSED - READY FOR DEPLOYMENT
|
||||||
|
|
||||||
|
Next Steps:
|
||||||
|
1. Deploy workflows: td wf push unification
|
||||||
|
2. Execute: td wf start unification unif_runner --session now
|
||||||
|
3. Monitor: td wf session <session_id>
|
||||||
|
```
|
||||||
|
|
||||||
|
**Failure Report:**
|
||||||
|
```
|
||||||
|
╔══════════════════════════════════════════════════════════════╗
|
||||||
|
║ ID UNIFICATION VALIDATION REPORT ║
|
||||||
|
╚══════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
|
[1/5] File Existence Check .......... ✅ PASS (15/15 files)
|
||||||
|
[2/5] Template Compliance Check ..... ❌ FAIL (2 errors)
|
||||||
|
❌ unif_runner.dig line 11: Uses call> instead of require>
|
||||||
|
FIX: Change "call>: dynmic_prep_creation.dig" to "require>: dynmic_prep_creation"
|
||||||
|
|
||||||
|
❌ stage_enrich.yml line 23: Incorrect column mapping
|
||||||
|
Expected: column: email_address_std (from col.name)
|
||||||
|
Found: column: email
|
||||||
|
FIX: Apply RULE 2 for staging tables
|
||||||
|
|
||||||
|
[3/5] Database & Table Existence .... ❌ FAIL (1 error)
|
||||||
|
❌ client_config.exclusion_list does NOT exist
|
||||||
|
FIX: td query -d client_config -t presto -w "CREATE TABLE IF NOT EXISTS exclusion_list (key_value VARCHAR, key_name VARCHAR, tbls ARRAY(VARCHAR), note VARCHAR)"
|
||||||
|
|
||||||
|
[4/5] Configuration Validation ...... ✅ PASS (8/8 checks)
|
||||||
|
[5/5] YAML Syntax Check ............. ✅ PASS (4/4 files)
|
||||||
|
|
||||||
|
╔══════════════════════════════════════════════════════════════╗
|
||||||
|
║ VALIDATION SUMMARY ║
|
||||||
|
╚══════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
|
Total Checks: 45
|
||||||
|
Passed: 42 ✅
|
||||||
|
Failed: 3 ❌
|
||||||
|
|
||||||
|
❌ VALIDATION FAILED - DO NOT DEPLOY
|
||||||
|
|
||||||
|
Required Actions:
|
||||||
|
1. Fix unif_runner.dig line 11 (use require> operator)
|
||||||
|
2. Fix stage_enrich.yml line 23 (use correct column mapping)
|
||||||
|
3. Create exclusion_list table
|
||||||
|
|
||||||
|
Re-run validation after fixes: /cdp-unification:unify-validate
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Agent Behavior
|
||||||
|
|
||||||
|
### STRICT MODE - ZERO TOLERANCE
|
||||||
|
|
||||||
|
1. **Stop at FIRST error** in each validation phase
|
||||||
|
2. **Provide EXACT fix command** for each error
|
||||||
|
3. **DO NOT proceed** if ANY validation fails
|
||||||
|
4. **Exit with error code** matching failure type
|
||||||
|
5. **Clear remediation steps** for each failure
|
||||||
|
|
||||||
|
### Tools to Use
|
||||||
|
|
||||||
|
- **Read tool**: Read all files for validation
|
||||||
|
- **MCP tools**: Check database and table existence
|
||||||
|
- **Grep tool**: Search for patterns in files
|
||||||
|
- **Bash tool**: Run validation scripts if needed
|
||||||
|
|
||||||
|
### DO NOT
|
||||||
|
|
||||||
|
- ❌ Skip any validation steps
|
||||||
|
- ❌ Proceed if errors found
|
||||||
|
- ❌ Suggest "it might work anyway"
|
||||||
|
- ❌ Auto-fix errors (show fix commands only)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Integration Requirements
|
||||||
|
|
||||||
|
This agent MUST be called:
|
||||||
|
1. **After** all files are generated by other agents
|
||||||
|
2. **Before** `td wf push` command
|
||||||
|
3. **Mandatory** in `/unify-setup` workflow
|
||||||
|
4. **Blocking** - deployment not allowed if fails
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**VALIDATION IS MANDATORY - NO EXCEPTIONS**
|
||||||
314
commands/unify-create-config.md
Normal file
314
commands/unify-create-config.md
Normal file
@@ -0,0 +1,314 @@
|
|||||||
|
---
|
||||||
|
name: unify-create-config
|
||||||
|
description: Generate core ID unification configuration files (unify.yml and id_unification.dig)
|
||||||
|
---
|
||||||
|
|
||||||
|
# Create Core Unification Configuration
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
I'll generate core ID unification configuration files using the **id-unification-creator** specialized agent.
|
||||||
|
|
||||||
|
This command creates **TD-COMPLIANT** unification files:
|
||||||
|
- ✅ **DYNAMIC CONFIGURATION** - Based on prep table analysis
|
||||||
|
- ✅ **METHOD-SPECIFIC** - Persistent_id OR canonical_id (never both)
|
||||||
|
- ✅ **REGIONAL ENDPOINTS** - Correct URL for your region
|
||||||
|
- ✅ **SCHEMA VALIDATION** - Prevents first-run failures
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Prerequisites
|
||||||
|
|
||||||
|
**REQUIRED**: Prep table configuration must exist:
|
||||||
|
- `unification/config/environment.yml` - Client configuration
|
||||||
|
- `unification/config/src_prep_params.yml` - Prep table mappings
|
||||||
|
|
||||||
|
If you haven't created these yet, run:
|
||||||
|
- `/cdp-unification:unify-create-prep` first, OR
|
||||||
|
- `/cdp-unification:unify-setup` for complete end-to-end setup
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What You Need to Provide
|
||||||
|
|
||||||
|
### 1. ID Method Selection
|
||||||
|
Choose ONE method:
|
||||||
|
|
||||||
|
**Option A: persistent_id (RECOMMENDED)**
|
||||||
|
- Stable IDs that persist across updates
|
||||||
|
- Better for customer data platforms
|
||||||
|
- Recommended for most use cases
|
||||||
|
- **Provide persistent_id name** (e.g., `td_claude_id`, `stable_customer_id`)
|
||||||
|
|
||||||
|
**Option B: canonical_id**
|
||||||
|
- Traditional approach with merge capabilities
|
||||||
|
- Good for legacy systems
|
||||||
|
- **Provide canonical_id name** (e.g., `master_customer_id`)
|
||||||
|
|
||||||
|
### 2. Update Strategy
|
||||||
|
- **Full Refresh**: Reprocess all data each time (`full_refresh: true`)
|
||||||
|
- **Incremental**: Process only new/updated records (`full_refresh: false`)
|
||||||
|
|
||||||
|
### 3. Regional Endpoint
|
||||||
|
Choose your Treasure Data region:
|
||||||
|
- **US**: https://api-cdp.treasuredata.com/unifications/workflow_call
|
||||||
|
- **EU**: https://api-cdp.eu01.treasuredata.com/unifications/workflow_call
|
||||||
|
- **Asia Pacific**: https://api-cdp.ap02.treasuredata.com/unifications/workflow_call
|
||||||
|
- **Japan**: https://api-cdp.treasuredata.co.jp/unifications/workflow_call
|
||||||
|
|
||||||
|
### 4. Unification Name
|
||||||
|
- Name for this unification project (e.g., `claude`, `customer_360`)
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What I'll Do
|
||||||
|
|
||||||
|
### Step 1: Validate Prerequisites
|
||||||
|
I'll check that these files exist:
|
||||||
|
- `unification/config/environment.yml`
|
||||||
|
- `unification/config/src_prep_params.yml`
|
||||||
|
|
||||||
|
And extract:
|
||||||
|
- Client short name
|
||||||
|
- Unified input table name
|
||||||
|
- All prep table configurations with column mappings
|
||||||
|
|
||||||
|
### Step 2: Extract Key Information
|
||||||
|
I'll parse `src_prep_params.yml` to identify:
|
||||||
|
- All unique `alias_as` column names
|
||||||
|
- Key types: email, phone, td_client_id, td_global_id, customer_id, etc.
|
||||||
|
- Complete list of available keys for `merge_by_keys`
|
||||||
|
|
||||||
|
### Step 3: Generate unification/config/unify.yml
|
||||||
|
I'll create:
|
||||||
|
```yaml
|
||||||
|
name: {unif_name}
|
||||||
|
|
||||||
|
keys:
|
||||||
|
- name: email
|
||||||
|
invalid_texts: ['']
|
||||||
|
- name: td_client_id
|
||||||
|
invalid_texts: ['']
|
||||||
|
- name: phone
|
||||||
|
invalid_texts: ['']
|
||||||
|
# ... ALL detected key types
|
||||||
|
|
||||||
|
tables:
|
||||||
|
- database: ${client_short_name}_${stg}
|
||||||
|
table: ${globals.unif_input_tbl}
|
||||||
|
incremental_columns: [time]
|
||||||
|
key_columns:
|
||||||
|
- {column: email, key: email}
|
||||||
|
- {column: td_client_id, key: td_client_id}
|
||||||
|
- {column: phone, key: phone}
|
||||||
|
# ... ALL alias_as columns mapped
|
||||||
|
|
||||||
|
# ONLY ONE of these sections (based on your selection):
|
||||||
|
persistent_ids:
|
||||||
|
- name: {persistent_id_name}
|
||||||
|
merge_by_keys: [email, td_client_id, phone, ...]
|
||||||
|
merge_iterations: 15
|
||||||
|
|
||||||
|
# OR
|
||||||
|
|
||||||
|
canonical_ids:
|
||||||
|
- name: {canonical_id_name}
|
||||||
|
merge_by_keys: [email, td_client_id, phone, ...]
|
||||||
|
merge_iterations: 15
|
||||||
|
```
|
||||||
|
|
||||||
|
### Step 4: Validate and Update Schema (CRITICAL)
|
||||||
|
I'll prevent first-run failures by:
|
||||||
|
1. Reading `unify.yml` to extract `merge_by_keys` list
|
||||||
|
2. Reading `queries/create_schema.sql` to check existing columns
|
||||||
|
3. Comparing required vs existing columns
|
||||||
|
4. Updating `create_schema.sql` if missing columns:
|
||||||
|
- Add all keys from `merge_by_keys` as varchar
|
||||||
|
- Add source, time, ingest_time columns
|
||||||
|
- Update BOTH table definitions (main and tmp)
|
||||||
|
|
||||||
|
### Step 5: Generate unification/id_unification.dig
|
||||||
|
I'll create:
|
||||||
|
```yaml
|
||||||
|
timezone: UTC
|
||||||
|
|
||||||
|
_export:
|
||||||
|
!include : config/environment.yml
|
||||||
|
!include : config/src_prep_params.yml
|
||||||
|
|
||||||
|
+call_unification:
|
||||||
|
http_call>: {REGIONAL_ENDPOINT_URL}
|
||||||
|
headers:
|
||||||
|
- authorization: ${secret:td.apikey}
|
||||||
|
- content-type: application/json
|
||||||
|
method: POST
|
||||||
|
retry: true
|
||||||
|
content_format: json
|
||||||
|
content:
|
||||||
|
run_persistent_ids: {true/false} # ONLY if persistent_id
|
||||||
|
run_canonical_ids: {true/false} # ONLY if canonical_id
|
||||||
|
run_enrichments: true
|
||||||
|
run_master_tables: true
|
||||||
|
full_refresh: {true/false}
|
||||||
|
keep_debug_tables: true
|
||||||
|
unification:
|
||||||
|
!include : config/unify.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Expected Output
|
||||||
|
|
||||||
|
### Files Created
|
||||||
|
```
|
||||||
|
unification/
|
||||||
|
├── config/
|
||||||
|
│ └── unify.yml ✓ Dynamic configuration
|
||||||
|
├── queries/
|
||||||
|
│ └── create_schema.sql ✓ Updated with all columns
|
||||||
|
└── id_unification.dig ✓ Core unification workflow
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example unify.yml (persistent_id method)
|
||||||
|
```yaml
|
||||||
|
name: customer_360
|
||||||
|
|
||||||
|
keys:
|
||||||
|
- name: email
|
||||||
|
invalid_texts: ['']
|
||||||
|
- name: td_client_id
|
||||||
|
invalid_texts: ['']
|
||||||
|
valid_regexp: '^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$'
|
||||||
|
|
||||||
|
tables:
|
||||||
|
- database: ${client_short_name}_${stg}
|
||||||
|
table: ${globals.unif_input_tbl}
|
||||||
|
incremental_columns: [time]
|
||||||
|
key_columns:
|
||||||
|
- {column: email, key: email}
|
||||||
|
- {column: td_client_id, key: td_client_id}
|
||||||
|
|
||||||
|
persistent_ids:
|
||||||
|
- name: td_claude_id
|
||||||
|
merge_by_keys: [email, td_client_id]
|
||||||
|
merge_iterations: 15
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example id_unification.dig (US region, incremental)
|
||||||
|
```yaml
|
||||||
|
timezone: UTC
|
||||||
|
|
||||||
|
_export:
|
||||||
|
!include : config/environment.yml
|
||||||
|
!include : config/src_prep_params.yml
|
||||||
|
|
||||||
|
+call_unification:
|
||||||
|
http_call>: https://api-cdp.treasuredata.com/unifications/workflow_call
|
||||||
|
headers:
|
||||||
|
- authorization: ${secret:td.apikey}
|
||||||
|
- content-type: application/json
|
||||||
|
method: POST
|
||||||
|
retry: true
|
||||||
|
content_format: json
|
||||||
|
content:
|
||||||
|
run_persistent_ids: true
|
||||||
|
run_enrichments: true
|
||||||
|
run_master_tables: true
|
||||||
|
full_refresh: false
|
||||||
|
keep_debug_tables: true
|
||||||
|
unification:
|
||||||
|
!include : config/unify.yml
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Critical Requirements
|
||||||
|
|
||||||
|
### ✅ Dynamic Configuration
|
||||||
|
- All keys detected from `src_prep_params.yml`
|
||||||
|
- All column mappings from prep analysis
|
||||||
|
- Method-specific configuration (never both)
|
||||||
|
|
||||||
|
### ⚠️ Schema Completeness
|
||||||
|
- `create_schema.sql` MUST contain ALL columns from `merge_by_keys`
|
||||||
|
- Prevents "column not found" errors on first run
|
||||||
|
- Updates both main and tmp table definitions
|
||||||
|
|
||||||
|
### ⚠️ Config File Inclusion
|
||||||
|
- `id_unification.dig` MUST include BOTH config files in `_export`:
|
||||||
|
- `environment.yml` - For `${client_short_name}_${stg}`
|
||||||
|
- `src_prep_params.yml` - For `${globals.unif_input_tbl}`
|
||||||
|
|
||||||
|
### ⚠️ Regional Endpoint
|
||||||
|
- Must use exact URL for selected region
|
||||||
|
- Different endpoints for US, EU, Asia Pacific, Japan
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Checklist
|
||||||
|
|
||||||
|
Before completing, I'll verify:
|
||||||
|
- [ ] unify.yml contains all detected key types
|
||||||
|
- [ ] key_columns section maps ALL alias_as columns
|
||||||
|
- [ ] Only ONE ID method section exists
|
||||||
|
- [ ] merge_by_keys includes ALL available keys
|
||||||
|
- [ ] **CRITICAL**: create_schema.sql contains ALL columns from merge_by_keys
|
||||||
|
- [ ] **CRITICAL**: Both table definitions updated (main and tmp)
|
||||||
|
- [ ] id_unification.dig has correct regional endpoint
|
||||||
|
- [ ] **CRITICAL**: _export includes BOTH config files
|
||||||
|
- [ ] Workflow flags match selected method only
|
||||||
|
- [ ] Proper TD YAML/DIG syntax
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
All generated files will:
|
||||||
|
- ✅ **TD-COMPLIANT** - Work without modification in TD
|
||||||
|
- ✅ **DYNAMICALLY CONFIGURED** - Based on actual prep analysis
|
||||||
|
- ✅ **METHOD-ACCURATE** - Exact implementation of selected method
|
||||||
|
- ✅ **REGIONALLY CORRECT** - Proper endpoint for region
|
||||||
|
- ✅ **SCHEMA-COMPLETE** - All required columns present
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
After creating core config, you can:
|
||||||
|
1. **Test unification workflow**: `dig run unification/id_unification.dig`
|
||||||
|
2. **Add enrichment**: Use `/cdp-unification:unify-setup` to add staging enrichment
|
||||||
|
3. **Create main orchestrator**: Combine prep + unification + enrichment
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
**Ready to create core unification config?** Please provide:
|
||||||
|
|
||||||
|
1. **ID Method**:
|
||||||
|
- Choose: `persistent_id` or `canonical_id`
|
||||||
|
- Provide ID name: e.g., `td_claude_id`
|
||||||
|
|
||||||
|
2. **Update Strategy**:
|
||||||
|
- Choose: `incremental` or `full_refresh`
|
||||||
|
|
||||||
|
3. **Regional Endpoint**:
|
||||||
|
- Choose: `US`, `EU`, `Asia Pacific`, or `Japan`
|
||||||
|
|
||||||
|
4. **Unification Name**:
|
||||||
|
- e.g., `customer_360`, `claude`
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```
|
||||||
|
ID Method: persistent_id
|
||||||
|
ID Name: td_claude_id
|
||||||
|
Update Strategy: incremental
|
||||||
|
Region: US
|
||||||
|
Unification Name: customer_360
|
||||||
|
```
|
||||||
|
|
||||||
|
I'll call the **id-unification-creator** agent to generate all core unification files.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Let's create your unification configuration!**
|
||||||
233
commands/unify-create-prep.md
Normal file
233
commands/unify-create-prep.md
Normal file
@@ -0,0 +1,233 @@
|
|||||||
|
---
|
||||||
|
name: unify-create-prep
|
||||||
|
description: Generate prep table creation files and configuration for ID unification
|
||||||
|
---
|
||||||
|
|
||||||
|
# Create Prep Table Configuration
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
I'll generate prep table creation files and configuration using the **dynamic-prep-creation** specialized agent.
|
||||||
|
|
||||||
|
This command creates **PRODUCTION-READY** prep table files:
|
||||||
|
- ⚠️ **EXACT TEMPLATES** - No modifications allowed
|
||||||
|
- ⚠️ **ZERO CHANGES** - Character-for-character accuracy
|
||||||
|
- ✅ **GENERIC FILES** - Reusable across all projects
|
||||||
|
- ✅ **DYNAMIC CONFIGURATION** - Adapts to your table structure
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What You Need to Provide
|
||||||
|
|
||||||
|
### 1. Table Analysis Results
|
||||||
|
If you've already run key extraction:
|
||||||
|
- Provide the list of **included tables** with their user identifier columns
|
||||||
|
- I can use the results from `/cdp-unification:unify-extract-keys`
|
||||||
|
|
||||||
|
OR provide directly:
|
||||||
|
- **Source tables**: database.table_name format
|
||||||
|
- **User identifier columns**: For each table, which columns contain identifiers
|
||||||
|
|
||||||
|
### 2. Client Configuration
|
||||||
|
- **Client short name**: Your client identifier (e.g., `mck`, `client_name`)
|
||||||
|
- **Database suffixes**:
|
||||||
|
- Source database suffix (default: `src`)
|
||||||
|
- Staging database suffix (default: `stg`)
|
||||||
|
- Lookup database (default: `config`)
|
||||||
|
|
||||||
|
### 3. Column Mappings
|
||||||
|
For each table, specify which columns to include and their unified aliases:
|
||||||
|
- **Email columns** → alias: `email`
|
||||||
|
- **Phone columns** → alias: `phone`
|
||||||
|
- **Customer ID columns** → alias: `customer_id`
|
||||||
|
- **TD Client ID** → alias: `td_client_id`
|
||||||
|
- **TD Global ID** → alias: `td_global_id`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What I'll Do
|
||||||
|
|
||||||
|
### Step 1: Create Directory Structure
|
||||||
|
I'll create:
|
||||||
|
- `unification/config/` directory
|
||||||
|
- `unification/queries/` directory
|
||||||
|
|
||||||
|
### Step 2: Generate Generic Files (EXACT TEMPLATES)
|
||||||
|
I'll create these files with **ZERO MODIFICATIONS**:
|
||||||
|
|
||||||
|
**⚠️ `unification/dynmic_prep_creation.dig`** (EXACT filename - no 'a' in dynmic)
|
||||||
|
- Generic prep workflow
|
||||||
|
- Handles schema creation, table looping, and data insertion
|
||||||
|
- Uses variables from config files
|
||||||
|
|
||||||
|
**⚠️ `unification/queries/create_schema.sql`**
|
||||||
|
- Generic schema creation for unified input table
|
||||||
|
- Creates both main and tmp tables
|
||||||
|
|
||||||
|
**⚠️ `unification/queries/loop_on_tables.sql`**
|
||||||
|
- Complex production SQL for dynamic table processing
|
||||||
|
- Generates prep table SQL and unified input table SQL
|
||||||
|
- Handles incremental logic and deduplication
|
||||||
|
|
||||||
|
**⚠️ `unification/queries/unif_input_tbl.sql`**
|
||||||
|
- DSAR processing and data cleaning
|
||||||
|
- Exclusion list management for masked data
|
||||||
|
- Dynamic column detection and insertion
|
||||||
|
|
||||||
|
### Step 3: Generate Dynamic Configuration Files
|
||||||
|
|
||||||
|
**`unification/config/environment.yml`**
|
||||||
|
```yaml
|
||||||
|
client_short_name: {your_client_name}
|
||||||
|
src: src
|
||||||
|
stg: stg
|
||||||
|
gld: gld
|
||||||
|
lkup: references
|
||||||
|
```
|
||||||
|
|
||||||
|
**`unification/config/src_prep_params.yml`**
|
||||||
|
- Dynamic table configuration based on your table analysis
|
||||||
|
- Column mappings with unified aliases
|
||||||
|
- Prep table naming conventions
|
||||||
|
|
||||||
|
### Step 4: Dynamic Column Detection (CRITICAL)
|
||||||
|
For `unif_input_tbl.sql`, I'll:
|
||||||
|
1. Query Treasure Data schema: `information_schema.columns`
|
||||||
|
2. Detect all columns besides email, phone, source, ingest_time, time
|
||||||
|
3. Auto-generate column list for data_cleaned CTE
|
||||||
|
4. Replace placeholder with actual columns
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Expected Output
|
||||||
|
|
||||||
|
### Generic Files (EXACT - NO CHANGES)
|
||||||
|
```
|
||||||
|
unification/
|
||||||
|
├── dynmic_prep_creation.dig ⚠️ EXACT filename
|
||||||
|
├── queries/
|
||||||
|
│ ├── create_schema.sql ⚠️ EXACT content
|
||||||
|
│ ├── loop_on_tables.sql ⚠️ EXACT content
|
||||||
|
│ └── unif_input_tbl.sql ⚠️ WITH dynamic columns
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dynamic Configuration Files
|
||||||
|
```
|
||||||
|
unification/config/
|
||||||
|
├── environment.yml ✓ Client-specific
|
||||||
|
└── src_prep_params.yml ✓ Table-specific
|
||||||
|
```
|
||||||
|
|
||||||
|
### Example src_prep_params.yml Structure
|
||||||
|
```yaml
|
||||||
|
globals:
|
||||||
|
unif_input_tbl: unif_input
|
||||||
|
|
||||||
|
prep_tbls:
|
||||||
|
- src_tbl: user_events
|
||||||
|
src_db: ${client_short_name}_${stg}
|
||||||
|
snk_db: ${client_short_name}_${stg}
|
||||||
|
snk_tbl: ${src_tbl}_prep
|
||||||
|
columns:
|
||||||
|
- col:
|
||||||
|
name: user_email
|
||||||
|
alias_as: email
|
||||||
|
- col:
|
||||||
|
name: td_client_id
|
||||||
|
alias_as: td_client_id
|
||||||
|
|
||||||
|
- src_tbl: customers
|
||||||
|
src_db: ${client_short_name}_${stg}
|
||||||
|
snk_db: ${client_short_name}_${stg}
|
||||||
|
snk_tbl: ${src_tbl}_prep
|
||||||
|
columns:
|
||||||
|
- col:
|
||||||
|
name: email
|
||||||
|
alias_as: email
|
||||||
|
- col:
|
||||||
|
name: customer_id
|
||||||
|
alias_as: customer_id
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Critical Requirements
|
||||||
|
|
||||||
|
### ⚠️ NEVER MODIFY GENERIC FILES
|
||||||
|
- **dynmic_prep_creation.dig**: EXACT template, character-for-character
|
||||||
|
- **create_schema.sql**: EXACT SQL, no changes
|
||||||
|
- **loop_on_tables.sql**: EXACT complex SQL, no modifications
|
||||||
|
- **unif_input_tbl.sql**: EXACT template + dynamic column replacement
|
||||||
|
|
||||||
|
### ✅ DYNAMIC CONFIGURATION ONLY
|
||||||
|
- **environment.yml**: Client-specific variables
|
||||||
|
- **src_prep_params.yml**: Table-specific mappings
|
||||||
|
|
||||||
|
### 🚨 CRITICAL FILENAME
|
||||||
|
- **MUST be "dynmic_prep_creation.dig"** (NO 'a' in dynmic)
|
||||||
|
- This is intentional - production systems expect this exact name
|
||||||
|
|
||||||
|
### 🚨 NO TIME COLUMN
|
||||||
|
- **NEVER ADD** `time` column to src_prep_params.yml
|
||||||
|
- Time is auto-generated by SQL template
|
||||||
|
- Only include actual identifier columns
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Checklist
|
||||||
|
|
||||||
|
Before completing, I'll verify:
|
||||||
|
- [ ] File named "dynmic_prep_creation.dig" exists
|
||||||
|
- [ ] Content matches template character-for-character
|
||||||
|
- [ ] All variable placeholders preserved
|
||||||
|
- [ ] Queries folder contains exact SQL files
|
||||||
|
- [ ] Config folder contains YAML files
|
||||||
|
- [ ] Dynamic columns inserted in unif_input_tbl.sql
|
||||||
|
- [ ] No time column in src_prep_params.yml
|
||||||
|
- [ ] All directories created
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
All generated files will:
|
||||||
|
- ✅ **EXACT TEMPLATES** - Character-for-character accuracy
|
||||||
|
- ✅ **PRODUCTION-READY** - Deployable to TD without changes
|
||||||
|
- ✅ **DYNAMIC CONFIGURATION** - Adapts to table structure
|
||||||
|
- ✅ **DSAR COMPLIANT** - Includes exclusion list processing
|
||||||
|
- ✅ **INCREMENTAL PROCESSING** - Supports time-based updates
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
After prep creation, you can:
|
||||||
|
1. **Test prep workflow**: `dig run unification/dynmic_prep_creation.dig`
|
||||||
|
2. **Create unification config**: Use `/cdp-unification:unify-create-config`
|
||||||
|
3. **Complete full setup**: Use `/cdp-unification:unify-setup`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
**Ready to create prep tables?** Please provide:
|
||||||
|
|
||||||
|
1. **Table list with columns**:
|
||||||
|
```
|
||||||
|
Table: analytics.user_events
|
||||||
|
Columns: user_email (email), td_client_id (td_client_id)
|
||||||
|
|
||||||
|
Table: crm.customers
|
||||||
|
Columns: email (email), customer_id (customer_id)
|
||||||
|
```
|
||||||
|
|
||||||
|
2. **Client configuration**:
|
||||||
|
```
|
||||||
|
Client short name: mck
|
||||||
|
```
|
||||||
|
|
||||||
|
I'll call the **dynamic-prep-creation** agent to generate all prep files with exact templates.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Let's create your prep table configuration!**
|
||||||
191
commands/unify-extract-keys.md
Normal file
191
commands/unify-extract-keys.md
Normal file
@@ -0,0 +1,191 @@
|
|||||||
|
---
|
||||||
|
name: unify-extract-keys
|
||||||
|
description: Extract and validate user identifier columns from tables using live Treasure Data analysis
|
||||||
|
---
|
||||||
|
|
||||||
|
# Extract and Validate User Identifiers
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
I'll analyze your Treasure Data tables to extract and validate user identifier columns using the **unif-keys-extractor** specialized agent.
|
||||||
|
|
||||||
|
This command performs **ZERO-TOLERANCE** identifier extraction:
|
||||||
|
- ❌ **NO GUESSING** - Only uses real Treasure Data MCP tools
|
||||||
|
- ❌ **NO ASSUMPTIONS** - Every table is analyzed with live data
|
||||||
|
- ✅ **STRICT VALIDATION** - Only includes tables with actual user identifiers
|
||||||
|
- ✅ **COMPREHENSIVE ANALYSIS** - 3 SQL experts review and priority recommendations
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What You Need to Provide
|
||||||
|
|
||||||
|
### Table List
|
||||||
|
Provide the tables you want to analyze for ID unification:
|
||||||
|
- **Format**: `database.table_name`
|
||||||
|
- **Example**: `analytics.user_events`, `crm.customers`, `web.pageviews`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What I'll Do
|
||||||
|
|
||||||
|
### Step 1: Schema Extraction (MANDATORY)
|
||||||
|
For each table, I'll:
|
||||||
|
- Call `mcp__mcc_treasuredata__describe_table(table, database)`
|
||||||
|
- Extract EXACT column names and data types
|
||||||
|
- Identify tables that are inaccessible
|
||||||
|
|
||||||
|
### Step 2: User Identifier Detection (STRICT MATCHING)
|
||||||
|
I'll scan for valid user identifier columns:
|
||||||
|
|
||||||
|
**✅ VALID USER IDENTIFIERS:**
|
||||||
|
- **Email columns**: email, email_std, email_address, user_email, customer_email
|
||||||
|
- **Phone columns**: phone, phone_std, phone_number, mobile_phone, customer_phone
|
||||||
|
- **User ID columns**: user_id, customer_id, account_id, member_id, uid, user_uuid
|
||||||
|
- **Identity columns**: profile_id, identity_id, cognito_identity_userid
|
||||||
|
- **Cookie/Device IDs**: td_client_id, td_global_id, td_ssc_id, cookie_id, device_id
|
||||||
|
|
||||||
|
**❌ NOT USER IDENTIFIERS (EXCLUDED):**
|
||||||
|
- System columns: id, created_at, updated_at, load_timestamp
|
||||||
|
- Campaign columns: campaign_id, message_id
|
||||||
|
- Product columns: product_id, sku, variant_id
|
||||||
|
- Complex types: array, map, json columns
|
||||||
|
|
||||||
|
### Step 3: Exclusion Validation (CRITICAL)
|
||||||
|
For tables WITHOUT user identifiers, I'll:
|
||||||
|
- Document the exclusion reason
|
||||||
|
- List available columns for transparency
|
||||||
|
- Explain why the table doesn't qualify
|
||||||
|
|
||||||
|
### Step 4: Min/Max Data Analysis (INCLUDED TABLES ONLY)
|
||||||
|
For tables WITH user identifiers, I'll:
|
||||||
|
- Query actual data: `SELECT MIN(column), MAX(column) FROM table`
|
||||||
|
- Validate data patterns and formats
|
||||||
|
- Assess data quality
|
||||||
|
|
||||||
|
### Step 5: 3 SQL Experts Analysis
|
||||||
|
I'll provide structured analysis from three perspectives:
|
||||||
|
1. **Data Pattern Analyst**: Reviews actual min/max values and data quality
|
||||||
|
2. **Cross-Table Relationship Analyst**: Maps identifier relationships across tables
|
||||||
|
3. **Priority Assessment Specialist**: Ranks identifiers by stability and coverage
|
||||||
|
|
||||||
|
### Step 6: Priority Recommendations
|
||||||
|
I'll provide:
|
||||||
|
- Recommended priority ordering (TD standard)
|
||||||
|
- Reasoning for each recommendation
|
||||||
|
- Compatibility assessment across tables
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Expected Output
|
||||||
|
|
||||||
|
### Key Extraction Results Table
|
||||||
|
```
|
||||||
|
| database_name | table_name | column_name | data_type | identifier_type | min_value | max_value |
|
||||||
|
|---------------|------------|-------------|-----------|-----------------|-----------|-----------|
|
||||||
|
| analytics | user_events| user_email | varchar | email | a@test.com| z@test.com|
|
||||||
|
| analytics | user_events| td_client_id| varchar | cookie_id | 00000000-.| ffffffff-.|
|
||||||
|
| crm | customers | email | varchar | email | admin@... | user@... |
|
||||||
|
```
|
||||||
|
|
||||||
|
### Exclusion Documentation
|
||||||
|
```
|
||||||
|
## Tables EXCLUDED from ID Unification:
|
||||||
|
|
||||||
|
- **analytics.product_catalog**: No user identifier columns found
|
||||||
|
- Available columns: [product_id, sku, product_name, category, price]
|
||||||
|
- Exclusion reason: Contains only product metadata - no PII
|
||||||
|
- Classification: Non-PII table
|
||||||
|
```
|
||||||
|
|
||||||
|
### Validation Summary
|
||||||
|
```
|
||||||
|
## Analysis Summary:
|
||||||
|
- **Tables Analyzed**: 5
|
||||||
|
- **Tables INCLUDED**: 3 (contain user identifiers)
|
||||||
|
- **Tables EXCLUDED**: 2 (no user identifiers)
|
||||||
|
- **User Identifier Columns Found**: 8
|
||||||
|
```
|
||||||
|
|
||||||
|
### 3 SQL Experts Analysis
|
||||||
|
```
|
||||||
|
**Expert 1 - Data Pattern Analyst:**
|
||||||
|
- Email columns show valid format patterns across 2 tables
|
||||||
|
- td_client_id shows UUID format with good coverage
|
||||||
|
- Data quality: High (95%+ non-null for email)
|
||||||
|
|
||||||
|
**Expert 2 - Cross-Table Relationship Analyst:**
|
||||||
|
- Email appears in analytics.user_events and crm.customers (primary link)
|
||||||
|
- td_client_id unique to analytics.user_events (secondary ID)
|
||||||
|
- Recommendation: Email as primary key for unification
|
||||||
|
|
||||||
|
**Expert 3 - Priority Assessment Specialist:**
|
||||||
|
- Priority 1: email (stable, cross-table presence, good coverage)
|
||||||
|
- Priority 2: td_client_id (system-generated, analytics-specific)
|
||||||
|
- Recommended merge_by_keys: [email, td_client_id]
|
||||||
|
```
|
||||||
|
|
||||||
|
### Priority Recommendations (TD Standard)
|
||||||
|
```
|
||||||
|
Recommended Priority Order (TD Standard):
|
||||||
|
1. email - Stable identifier across multiple tables with high coverage
|
||||||
|
2. td_client_id - System-generated ID for analytics tracking
|
||||||
|
3. phone - Secondary contact identifier (if available)
|
||||||
|
|
||||||
|
EXCLUDED Identifiers (Not User-Related):
|
||||||
|
- product_id - Product reference, not user identifier
|
||||||
|
- campaign_id - Campaign metadata, not user-specific
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Gates
|
||||||
|
|
||||||
|
I'll pass through these mandatory validation gates:
|
||||||
|
- ✅ **GATE 1**: Schema extracted for all accessible tables
|
||||||
|
- ✅ **GATE 2**: Tables classified into INCLUSION/EXCLUSION lists
|
||||||
|
- ✅ **GATE 3**: All exclusions justified and documented
|
||||||
|
- ✅ **GATE 4**: Real data analysis completed for included columns
|
||||||
|
- ✅ **GATE 5**: 3 SQL experts analysis completed
|
||||||
|
- ✅ **GATE 6**: Priority recommendations provided
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Next Steps
|
||||||
|
|
||||||
|
After key extraction, you can:
|
||||||
|
1. **Proceed with full setup**: Use `/cdp-unification:unify-setup` to continue with complete configuration
|
||||||
|
2. **Create prep tables**: Use `/cdp-unification:unify-create-prep` with the extracted keys
|
||||||
|
3. **Review and adjust**: Discuss the results and make adjustments to table selection
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Communication Pattern
|
||||||
|
|
||||||
|
I'll use **TD Copilot standard format**:
|
||||||
|
|
||||||
|
**Question**: Are these extracted user identifiers sufficient for your ID unification requirements?
|
||||||
|
|
||||||
|
**Suggestion**: I recommend using **email** as your primary unification key since it appears across multiple tables with good data quality.
|
||||||
|
|
||||||
|
**Check Point**: The analysis shows X tables with user identifiers and Y tables excluded. This provides comprehensive coverage for customer identity resolution.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
**Ready to extract user identifiers?** Please provide your table list:
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```
|
||||||
|
Please analyze these tables for ID unification:
|
||||||
|
- analytics.user_events
|
||||||
|
- crm.customers
|
||||||
|
- web.pageviews
|
||||||
|
- marketing.campaigns
|
||||||
|
```
|
||||||
|
|
||||||
|
I'll call the **unif-keys-extractor** agent to perform comprehensive analysis with ZERO-TOLERANCE validation.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Let's begin the analysis!**
|
||||||
200
commands/unify-setup.md
Normal file
200
commands/unify-setup.md
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
---
|
||||||
|
name: unify-setup
|
||||||
|
description: Complete end-to-end ID unification setup from table analysis to deployment
|
||||||
|
---
|
||||||
|
|
||||||
|
# Complete ID Unification Setup
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
I'll guide you through the complete ID unification setup process for Treasure Data CDP. This is an interactive, end-to-end workflow that will:
|
||||||
|
|
||||||
|
1. **Extract and validate user identifiers** from your tables
|
||||||
|
2. **Help you choose the right ID method** (canonical_id vs persistent_id)
|
||||||
|
3. **Generate prep table configurations** for data standardization
|
||||||
|
4. **Create core unification files** (unify.yml and id_unification.dig)
|
||||||
|
5. **Set up staging enrichment** for post-unification processing
|
||||||
|
6. **Create orchestration workflow** (unif_runner.dig) to run everything in sequence
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What You Need to Provide
|
||||||
|
|
||||||
|
### 1. Table List
|
||||||
|
Please provide the list of tables you want to include in ID unification:
|
||||||
|
- Format: `database.table_name` (e.g., `analytics.user_events`, `crm.customers`)
|
||||||
|
- I'll analyze each table using Treasure Data MCP tools to extract user identifiers
|
||||||
|
|
||||||
|
### 2. Client Configuration
|
||||||
|
- **Client short name**: Your client identifier (e.g., `mck`, `client`)
|
||||||
|
- **Unification name**: Name for this unification project (e.g., `claude`, `customer_360`)
|
||||||
|
- **Lookup/Config database suffix**: (default: `config`)
|
||||||
|
- Creates database: `${client_short_name}_${lookup_suffix}` (e.g., `client_config`)
|
||||||
|
- ⚠️ **I WILL CREATE THIS DATABASE** if it doesn't exist
|
||||||
|
|
||||||
|
### 3. ID Method Selection
|
||||||
|
I'll explain the options and help you choose:
|
||||||
|
- **persistent_id**: Stable IDs that persist across updates (recommended for most cases)
|
||||||
|
- **canonical_id**: Traditional approach with merge capabilities
|
||||||
|
|
||||||
|
### 4. Update Strategy
|
||||||
|
- **Incremental**: Process only new/updated records
|
||||||
|
- **Full Refresh**: Reprocess all data each time
|
||||||
|
|
||||||
|
### 5. Regional Endpoint
|
||||||
|
- **US**: https://api-cdp.treasuredata.com
|
||||||
|
- **EU**: https://api-cdp.eu01.treasuredata.com
|
||||||
|
- **Asia Pacific**: https://api-cdp.ap02.treasuredata.com
|
||||||
|
- **Japan**: https://api-cdp.treasuredata.co.jp
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What I'll Do
|
||||||
|
|
||||||
|
### Step 1: Extract and Validate Keys (via unif-keys-extractor agent)
|
||||||
|
I'll:
|
||||||
|
- Use Treasure Data MCP tools to analyze table schemas
|
||||||
|
- Extract user identifier columns (email, phone, td_client_id, etc.)
|
||||||
|
- Query sample data to validate identifier patterns
|
||||||
|
- Provide 3 SQL experts analysis of key relationships
|
||||||
|
- Recommend priority ordering for unification keys
|
||||||
|
- Exclude tables without user identifiers
|
||||||
|
|
||||||
|
### Step 2: Configuration Guidance
|
||||||
|
I'll:
|
||||||
|
- Explain canonical_id vs persistent_id concepts
|
||||||
|
- Recommend best approach for your use case
|
||||||
|
- Discuss incremental vs full refresh strategies
|
||||||
|
- Help you understand regional endpoint requirements
|
||||||
|
|
||||||
|
### Step 3: Generate Prep Tables (via dynamic-prep-creation agent)
|
||||||
|
I'll create:
|
||||||
|
- `unification/dynmic_prep_creation.dig` - Prep workflow
|
||||||
|
- `unification/queries/create_schema.sql` - Schema creation
|
||||||
|
- `unification/queries/loop_on_tables.sql` - Dynamic loop logic
|
||||||
|
- `unification/queries/unif_input_tbl.sql` - DSAR processing and data cleaning
|
||||||
|
- `unification/config/environment.yml` - Client configuration
|
||||||
|
- `unification/config/src_prep_params.yml` - Dynamic table mappings
|
||||||
|
|
||||||
|
### Step 4: Generate Core Unification (via id-unification-creator agent)
|
||||||
|
I'll create:
|
||||||
|
- `unification/config/unify.yml` - Unification configuration with keys and tables
|
||||||
|
- `unification/id_unification.dig` - Core unification workflow with HTTP API call
|
||||||
|
- Updated `unification/queries/create_schema.sql` - Schema with all required columns
|
||||||
|
|
||||||
|
### Step 5: Generate Staging Enrichment (via unification-staging-enricher agent)
|
||||||
|
I'll create:
|
||||||
|
- `unification/config/stage_enrich.yml` - Enrichment configuration
|
||||||
|
- `unification/enrich/queries/generate_join_query.sql` - Join query generation
|
||||||
|
- `unification/enrich/queries/execute_join_presto.sql` - Presto execution
|
||||||
|
- `unification/enrich/queries/execute_join_hive.sql` - Hive execution
|
||||||
|
- `unification/enrich/queries/enrich_tbl_creation.sql` - Table creation
|
||||||
|
- `unification/enrich_runner.dig` - Enrichment workflow
|
||||||
|
|
||||||
|
### Step 6: Create Main Orchestration
|
||||||
|
I'll create:
|
||||||
|
- `unification/unif_runner.dig` - Main workflow that calls:
|
||||||
|
- prep_creation → id_unification → enrichment (in sequence)
|
||||||
|
|
||||||
|
### Step 7: ⚠️ MANDATORY VALIDATION (NEW!)
|
||||||
|
**CRITICAL**: Before deployment, I MUST run comprehensive validation:
|
||||||
|
- `/cdp-unification:unify-validate` command
|
||||||
|
- Validates ALL files against exact templates
|
||||||
|
- Checks database and table existence
|
||||||
|
- Verifies configuration consistency
|
||||||
|
- **BLOCKS deployment if ANY validation fails**
|
||||||
|
|
||||||
|
**If validation FAILS:**
|
||||||
|
- I will show exact fix commands
|
||||||
|
- You must fix all errors
|
||||||
|
- Re-run validation until 100% pass
|
||||||
|
- Only then proceed to deployment
|
||||||
|
|
||||||
|
**If validation PASSES:**
|
||||||
|
- Proceed to deployment with confidence
|
||||||
|
- All files are production-ready
|
||||||
|
|
||||||
|
### Step 8: Deployment Guidance
|
||||||
|
I'll provide:
|
||||||
|
- Configuration summary
|
||||||
|
- Deployment instructions
|
||||||
|
- Operating guidelines
|
||||||
|
- Monitoring recommendations
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Interactive Workflow
|
||||||
|
|
||||||
|
I'll use the **TD Copilot communication pattern** throughout:
|
||||||
|
|
||||||
|
- **Question**: When I need your input or choice
|
||||||
|
- **Suggestion**: When I recommend a specific approach
|
||||||
|
- **Check Point**: When you should verify understanding
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Expected Output
|
||||||
|
|
||||||
|
### Files Created (All under `unification/` directory):
|
||||||
|
|
||||||
|
**Workflows:**
|
||||||
|
- `unif_runner.dig` - Main orchestration workflow
|
||||||
|
- `dynmic_prep_creation.dig` - Prep table creation
|
||||||
|
- `id_unification.dig` - Core unification
|
||||||
|
- `enrich_runner.dig` - Staging enrichment
|
||||||
|
|
||||||
|
**Configuration:**
|
||||||
|
- `config/environment.yml` - Client settings
|
||||||
|
- `config/src_prep_params.yml` - Prep table mappings
|
||||||
|
- `config/unify.yml` - Unification configuration
|
||||||
|
- `config/stage_enrich.yml` - Enrichment configuration
|
||||||
|
|
||||||
|
**SQL Templates:**
|
||||||
|
- `queries/create_schema.sql` - Schema creation
|
||||||
|
- `queries/loop_on_tables.sql` - Dynamic loop logic
|
||||||
|
- `queries/unif_input_tbl.sql` - DSAR and data cleaning
|
||||||
|
- `enrich/queries/generate_join_query.sql` - Join generation
|
||||||
|
- `enrich/queries/execute_join_presto.sql` - Presto execution
|
||||||
|
- `enrich/queries/execute_join_hive.sql` - Hive execution
|
||||||
|
- `enrich/queries/enrich_tbl_creation.sql` - Table creation
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
All generated files will:
|
||||||
|
- ✅ Be TD-compliant and deployment-ready
|
||||||
|
- ✅ Use exact templates from documentation
|
||||||
|
- ✅ Include comprehensive error handling
|
||||||
|
- ✅ Follow TD Copilot standards
|
||||||
|
- ✅ Work without modification in Treasure Data
|
||||||
|
- ✅ Support incremental processing
|
||||||
|
- ✅ Include DSAR processing
|
||||||
|
- ✅ Generate proper master tables
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Getting Started
|
||||||
|
|
||||||
|
**Ready to begin?** Please provide:
|
||||||
|
|
||||||
|
1. Your table list (database.table_name format)
|
||||||
|
2. Client short name
|
||||||
|
3. Unification name
|
||||||
|
|
||||||
|
I'll start by analyzing your tables with the unif-keys-extractor agent to extract and validate user identifiers.
|
||||||
|
|
||||||
|
**Example:**
|
||||||
|
```
|
||||||
|
I want to set up ID unification for:
|
||||||
|
- analytics.user_events
|
||||||
|
- crm.customers
|
||||||
|
- web.pageviews
|
||||||
|
|
||||||
|
Client: mck
|
||||||
|
Unification name: customer_360
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Let's get started!**
|
||||||
194
commands/unify-validate.md
Normal file
194
commands/unify-validate.md
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
---
|
||||||
|
name: unify-validate
|
||||||
|
description: Validate all ID unification files against exact templates before deployment
|
||||||
|
---
|
||||||
|
|
||||||
|
# ID Unification Validation Command
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
**MANDATORY validation gate** that checks ALL generated unification files against exact templates from agent prompts. This prevents deployment of incorrect configurations.
|
||||||
|
|
||||||
|
**⚠️ CRITICAL**: This command MUST complete successfully before `td wf push` or workflow execution.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## What This Command Validates
|
||||||
|
|
||||||
|
### 1. File Existence Check
|
||||||
|
- ✅ `unification/unif_runner.dig` exists
|
||||||
|
- ✅ `unification/dynmic_prep_creation.dig` exists
|
||||||
|
- ✅ `unification/id_unification.dig` exists
|
||||||
|
- ✅ `unification/enrich_runner.dig` exists
|
||||||
|
- ✅ `unification/config/environment.yml` exists
|
||||||
|
- ✅ `unification/config/src_prep_params.yml` exists
|
||||||
|
- ✅ `unification/config/unify.yml` exists
|
||||||
|
- ✅ `unification/config/stage_enrich.yml` exists
|
||||||
|
- ✅ All SQL files in `unification/queries/` exist
|
||||||
|
- ✅ All SQL files in `unification/enrich/queries/` exist
|
||||||
|
|
||||||
|
### 2. Template Compliance Check
|
||||||
|
|
||||||
|
**unif_runner.dig Validation:**
|
||||||
|
- ✅ Uses `require>` operator (NOT `call>`)
|
||||||
|
- ✅ No `echo>` operators with subtasks
|
||||||
|
- ✅ Matches exact template from `/plugins/cdp-unification/prompt.md` lines 186-217
|
||||||
|
- ✅ Has `_error:` section with email_alert
|
||||||
|
- ✅ Includes both `config/environment.yml` and `config/src_prep_params.yml`
|
||||||
|
|
||||||
|
**stage_enrich.yml Validation:**
|
||||||
|
- ✅ RULE 1: `unif_input` table has `column` and `key` both using `alias_as`
|
||||||
|
- ✅ RULE 2: Staging tables have `column` using `col.name` and `key` using `alias_as`
|
||||||
|
- ✅ All key_columns match actual columns from `src_prep_params.yml`
|
||||||
|
- ✅ No template columns (like adobe_clickstream, loyalty_id_std)
|
||||||
|
- ✅ Table names match `src_tbl` (NO _prep suffix)
|
||||||
|
|
||||||
|
**enrich_runner.dig Validation:**
|
||||||
|
- ✅ Matches exact template from `unification-staging-enricher.md` lines 261-299
|
||||||
|
- ✅ Includes all 3 config files in `_export`
|
||||||
|
- ✅ Uses `td_for_each>` for dynamic execution
|
||||||
|
- ✅ Has Presto and Hive conditional execution
|
||||||
|
|
||||||
|
### 3. Database & Table Existence Check
|
||||||
|
- ✅ `${client_short_name}_${src}` database exists
|
||||||
|
- ✅ `${client_short_name}_${stg}` database exists
|
||||||
|
- ✅ `${client_short_name}_${gld}` database exists (if used)
|
||||||
|
- ✅ `${client_short_name}_${lkup}` database exists
|
||||||
|
- ✅ `cdp_unification_${unif_name}` database exists
|
||||||
|
- ✅ `${client_short_name}_${lkup}.exclusion_list` table exists
|
||||||
|
|
||||||
|
### 4. Configuration Validation
|
||||||
|
- ✅ All variables in `environment.yml` are defined
|
||||||
|
- ✅ All tables in `src_prep_params.yml` exist in source database
|
||||||
|
- ✅ All columns in `src_prep_params.yml` exist in source tables
|
||||||
|
- ✅ `unify.yml` merge_by_keys match `src_prep_params.yml` alias_as columns
|
||||||
|
- ✅ No undefined variables (${...})
|
||||||
|
|
||||||
|
### 5. YAML Syntax Check
|
||||||
|
- ✅ All YAML files have valid syntax
|
||||||
|
- ✅ Proper indentation (2 spaces)
|
||||||
|
- ✅ No tabs in YAML files
|
||||||
|
- ✅ All strings properly quoted where needed
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Validation Report Format
|
||||||
|
|
||||||
|
```
|
||||||
|
╔══════════════════════════════════════════════════════════════╗
|
||||||
|
║ ID UNIFICATION VALIDATION REPORT ║
|
||||||
|
╚══════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
|
[1/5] File Existence Check
|
||||||
|
✅ unification/unif_runner.dig
|
||||||
|
✅ unification/dynmic_prep_creation.dig
|
||||||
|
✅ unification/id_unification.dig
|
||||||
|
✅ unification/enrich_runner.dig
|
||||||
|
✅ unification/config/environment.yml
|
||||||
|
✅ unification/config/src_prep_params.yml
|
||||||
|
✅ unification/config/unify.yml
|
||||||
|
✅ unification/config/stage_enrich.yml
|
||||||
|
✅ 3/3 SQL files in queries/
|
||||||
|
✅ 4/4 SQL files in enrich/queries/
|
||||||
|
|
||||||
|
[2/5] Template Compliance Check
|
||||||
|
✅ unif_runner.dig uses require> operator
|
||||||
|
✅ unif_runner.dig has no echo> conflicts
|
||||||
|
✅ stage_enrich.yml RULE 1 compliant (unif_input table)
|
||||||
|
✅ stage_enrich.yml RULE 2 compliant (staging tables)
|
||||||
|
❌ stage_enrich.yml has incorrect mapping on line 23
|
||||||
|
Expected: column: email_address_std
|
||||||
|
Found: column: email
|
||||||
|
FIX: Update line 23 to use col.name from src_prep_params.yml
|
||||||
|
|
||||||
|
[3/5] Database & Table Existence
|
||||||
|
✅ client_src exists
|
||||||
|
✅ client_stg exists
|
||||||
|
✅ client_gld exists
|
||||||
|
✅ client_config exists
|
||||||
|
❌ client_config.exclusion_list does NOT exist
|
||||||
|
FIX: Run: td query -d client_config -t presto -w "CREATE TABLE IF NOT EXISTS exclusion_list (key_value VARCHAR, key_name VARCHAR, tbls ARRAY(VARCHAR), note VARCHAR)"
|
||||||
|
|
||||||
|
[4/5] Configuration Validation
|
||||||
|
✅ All variables defined in environment.yml
|
||||||
|
✅ Source table client_stg.snowflake_orders exists
|
||||||
|
✅ All columns exist in source table
|
||||||
|
✅ unify.yml keys match src_prep_params.yml
|
||||||
|
|
||||||
|
[5/5] YAML Syntax Check
|
||||||
|
✅ All YAML files have valid syntax
|
||||||
|
✅ Proper indentation
|
||||||
|
✅ No tabs found
|
||||||
|
|
||||||
|
╔══════════════════════════════════════════════════════════════╗
|
||||||
|
║ VALIDATION SUMMARY ║
|
||||||
|
╚══════════════════════════════════════════════════════════════╝
|
||||||
|
|
||||||
|
Total Checks: 45
|
||||||
|
Passed: 43 ✅
|
||||||
|
Failed: 2 ❌
|
||||||
|
|
||||||
|
❌ VALIDATION FAILED - DO NOT DEPLOY
|
||||||
|
|
||||||
|
Required Actions:
|
||||||
|
1. Fix stage_enrich.yml line 23 mapping
|
||||||
|
2. Create client_config.exclusion_list table
|
||||||
|
|
||||||
|
Re-run validation after fixes: /cdp-unification:unify-validate
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Error Codes
|
||||||
|
|
||||||
|
- **EXIT 0**: All validations passed ✅
|
||||||
|
- **EXIT 1**: File existence failures
|
||||||
|
- **EXIT 2**: Template compliance failures
|
||||||
|
- **EXIT 3**: Database/table missing
|
||||||
|
- **EXIT 4**: Configuration errors
|
||||||
|
- **EXIT 5**: YAML syntax errors
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Usage
|
||||||
|
|
||||||
|
**Standalone:**
|
||||||
|
```
|
||||||
|
/cdp-unification:unify-validate
|
||||||
|
```
|
||||||
|
|
||||||
|
**Auto-triggered in unify-setup** (MANDATORY step before deployment)
|
||||||
|
|
||||||
|
**Manual validation before deployment:**
|
||||||
|
```
|
||||||
|
cd unification
|
||||||
|
/cdp-unification:unify-validate
|
||||||
|
```
|
||||||
|
|
||||||
|
If validation PASSES → Proceed with `td wf push unification`
|
||||||
|
If validation FAILS → Fix errors and re-validate
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Integration with unify-setup
|
||||||
|
|
||||||
|
The `/unify-setup` command will automatically:
|
||||||
|
1. Generate all unification files
|
||||||
|
2. **RUN VALIDATION** (this command)
|
||||||
|
3. **BLOCK deployment** if validation fails
|
||||||
|
4. **Show fix instructions** for each error
|
||||||
|
5. **Auto-retry validation** after fixes
|
||||||
|
6. Only proceed to deployment after 100% validation success
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Success Criteria
|
||||||
|
|
||||||
|
✅ **ALL checks must pass** before deployment is allowed
|
||||||
|
✅ **No exceptions** - even 1 failure blocks deployment
|
||||||
|
✅ **Detailed error messages** with exact fix instructions
|
||||||
|
✅ **Auto-remediation suggestions** where possible
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
**Let's validate your unification files!**
|
||||||
81
plugin.lock.json
Normal file
81
plugin.lock.json
Normal file
@@ -0,0 +1,81 @@
|
|||||||
|
{
|
||||||
|
"$schema": "internal://schemas/plugin.lock.v1.json",
|
||||||
|
"pluginId": "gh:treasure-data/aps_claude_tools:plugins/cdp-unification",
|
||||||
|
"normalized": {
|
||||||
|
"repo": null,
|
||||||
|
"ref": "refs/tags/v20251128.0",
|
||||||
|
"commit": "9e882d69e4ed087936955aab3fa8e1a8025e0944",
|
||||||
|
"treeHash": "f01f420f1b92e54c6ec0a88580ace3adc59628800776400a35794d2a087b1c52",
|
||||||
|
"generatedAt": "2025-11-28T10:28:44.703748Z",
|
||||||
|
"toolVersion": "publish_plugins.py@0.2.0"
|
||||||
|
},
|
||||||
|
"origin": {
|
||||||
|
"remote": "git@github.com:zhongweili/42plugin-data.git",
|
||||||
|
"branch": "master",
|
||||||
|
"commit": "aa1497ed0949fd50e99e70d6324a29c5b34f9390",
|
||||||
|
"repoRoot": "/Users/zhongweili/projects/openmind/42plugin-data"
|
||||||
|
},
|
||||||
|
"manifest": {
|
||||||
|
"name": "cdp-unification",
|
||||||
|
"description": "Unify and consolidate data across multiple sources",
|
||||||
|
"version": null
|
||||||
|
},
|
||||||
|
"content": {
|
||||||
|
"files": [
|
||||||
|
{
|
||||||
|
"path": "README.md",
|
||||||
|
"sha256": "5faaf7156c72ceeb584f7feb3d35ad043ae8e5086c0acc8d12c82ab334730e2d"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "agents/unification-validator.md",
|
||||||
|
"sha256": "7231b2b96b8892083e7197031ca02dffc4acf1f2fce0733923a83705d15c559e"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "agents/id-unification-creator.md",
|
||||||
|
"sha256": "412b5ebd94a5b018a76843feac5ae551fc966a5bea05307706a01595c023c519"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "agents/unification-staging-enricher.md",
|
||||||
|
"sha256": "7385f65b36194919bb1f13f660f30398151cdc81084745224cfaf6cf35e44d8f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "agents/dynamic-prep-creation.md",
|
||||||
|
"sha256": "dcdd78abd6abb4d0ed55776bd7a8749b4f979c693539975b4dd0eaf55941a60f"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "agents/unif-keys-extractor.md",
|
||||||
|
"sha256": "b93615fcdfa6ffbbdf552269212fff95fd4c5d53b06954282db9597d2e33c6df"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": ".claude-plugin/plugin.json",
|
||||||
|
"sha256": "c72e26e4a91e2b5588aab21b69de54fb177c47c0bcb698cac825ad0d267a278c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "commands/unify-validate.md",
|
||||||
|
"sha256": "866f097118269c16892eb242ffe23f4b606e01e7d6bb66008930146b11b90a59"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "commands/unify-create-prep.md",
|
||||||
|
"sha256": "17f7a12cf16221ad889fd90a6d053617e9154702e0b42d2fe049bbc301f03296"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "commands/unify-setup.md",
|
||||||
|
"sha256": "a9138fadd894c3b52e1eff0dcf17300d11b832e4e5c6838f3b65098dc272a041"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "commands/unify-extract-keys.md",
|
||||||
|
"sha256": "83216f57dda1edbdeed71c0c246d9421f509bce74d958001c9e25854afc75a9c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"path": "commands/unify-create-config.md",
|
||||||
|
"sha256": "7151a9355ccedee42de1dcee2ee2dc53f09fb1b548f3e121942514d3784e6c60"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"dirSha256": "f01f420f1b92e54c6ec0a88580ace3adc59628800776400a35794d2a087b1c52"
|
||||||
|
},
|
||||||
|
"security": {
|
||||||
|
"scannedAt": null,
|
||||||
|
"scannerVersion": null,
|
||||||
|
"flags": []
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user