From 6ec3196ecc6d26379deda4224c1e5dfd571e0525 Mon Sep 17 00:00:00 2001 From: Zhongwei Li Date: Sun, 30 Nov 2025 08:48:52 +0800 Subject: [PATCH] Initial commit --- .claude-plugin/plugin.json | 34 + README.md | 3 + plugin.lock.json | 1813 +++++++ skills/ai-multimodal/.env.example | 97 + skills/ai-multimodal/SKILL.md | 357 ++ .../references/audio-processing.md | 373 ++ .../references/image-generation.md | 558 +++ .../references/video-analysis.md | 502 ++ .../references/vision-understanding.md | 483 ++ .../scripts/document_converter.py | 395 ++ .../scripts/gemini_batch_process.py | 480 ++ .../ai-multimodal/scripts/media_optimizer.py | 506 ++ skills/ai-multimodal/scripts/requirements.txt | 26 + .../scripts/tests/requirements.txt | 20 + .../scripts/tests/test_document_converter.py | 299 ++ .../tests/test_gemini_batch_process.py | 362 ++ .../scripts/tests/test_media_optimizer.py | 373 ++ skills/better-auth/SKILL.md | 204 + .../references/advanced-features.md | 553 ++ .../references/database-integration.md | 577 +++ .../references/email-password-auth.md | 416 ++ .../better-auth/references/oauth-providers.md | 430 ++ .../better-auth/scripts/better_auth_init.py | 521 ++ skills/better-auth/scripts/requirements.txt | 15 + .../scripts/tests/test_better_auth_init.py | 421 ++ skills/chrome-devtools/SKILL.md | 360 ++ .../chrome-devtools/references/cdp-domains.md | 694 +++ .../references/performance-guide.md | 940 ++++ .../references/puppeteer-reference.md | 953 ++++ skills/chrome-devtools/scripts/README.md | 213 + .../scripts/__tests__/selector.test.js | 210 + skills/chrome-devtools/scripts/click.js | 79 + skills/chrome-devtools/scripts/console.js | 75 + skills/chrome-devtools/scripts/evaluate.js | 49 + skills/chrome-devtools/scripts/fill.js | 72 + .../chrome-devtools/scripts/install-deps.sh | 181 + skills/chrome-devtools/scripts/install.sh | 83 + skills/chrome-devtools/scripts/navigate.js | 46 + skills/chrome-devtools/scripts/network.js | 102 + skills/chrome-devtools/scripts/package.json | 15 + skills/chrome-devtools/scripts/performance.js | 145 + skills/chrome-devtools/scripts/screenshot.js | 180 + skills/chrome-devtools/scripts/snapshot.js | 131 + skills/claude-code/SKILL.md | 181 + skills/claude-code/llms.txt | 135 + .../references/advanced-features.md | 399 ++ skills/claude-code/references/agent-skills.md | 414 ++ .../claude-code/references/api-reference.md | 498 ++ .../claude-code/references/best-practices.md | 447 ++ .../references/cicd-integration.md | 428 ++ .../claude-code/references/configuration.md | 480 ++ .../references/enterprise-features.md | 472 ++ .../claude-code/references/getting-started.md | 252 + .../references/hooks-and-plugins.md | 443 ++ .../claude-code/references/ide-integration.md | 316 ++ .../claude-code/references/mcp-integration.md | 386 ++ .../claude-code/references/slash-commands.md | 489 ++ .../claude-code/references/troubleshooting.md | 456 ++ skills/claude-code/skill.json | 6 + skills/code-review/SKILL.md | 140 + .../references/code-review-reception.md | 209 + .../references/requesting-code-review.md | 105 + .../verification-before-completion.md | 139 + skills/common/README.md | 120 + skills/common/api_key_helper.py | 300 ++ skills/databases/SKILL.md | 232 + .../references/mongodb-aggregation.md | 447 ++ skills/databases/references/mongodb-atlas.md | 465 ++ skills/databases/references/mongodb-crud.md | 408 ++ .../databases/references/mongodb-indexing.md | 442 ++ .../references/postgresql-administration.md | 594 +++ .../references/postgresql-performance.md | 527 ++ .../references/postgresql-psql-cli.md | 467 ++ .../references/postgresql-queries.md | 475 ++ skills/databases/scripts/db_backup.py | 502 ++ skills/databases/scripts/db_migrate.py | 414 ++ .../databases/scripts/db_performance_check.py | 444 ++ skills/databases/scripts/requirements.txt | 20 + .../databases/scripts/tests/coverage-db.json | 1 + .../databases/scripts/tests/requirements.txt | 4 + .../databases/scripts/tests/test_db_backup.py | 340 ++ .../scripts/tests/test_db_migrate.py | 277 + .../tests/test_db_performance_check.py | 370 ++ skills/debugging/defense-in-depth/SKILL.md | 130 + skills/debugging/root-cause-tracing/SKILL.md | 177 + .../root-cause-tracing/find-polluter.sh | 63 + .../systematic-debugging/CREATION-LOG.md | 119 + .../debugging/systematic-debugging/SKILL.md | 295 ++ .../systematic-debugging/test-academic.md | 14 + .../systematic-debugging/test-pressure-1.md | 58 + .../systematic-debugging/test-pressure-2.md | 68 + .../systematic-debugging/test-pressure-3.md | 69 + .../verification-before-completion/SKILL.md | 142 + skills/devops/.env.example | 76 + skills/devops/SKILL.md | 285 ++ skills/devops/references/browser-rendering.md | 305 ++ skills/devops/references/cloudflare-d1-kv.md | 123 + .../devops/references/cloudflare-platform.md | 271 + .../references/cloudflare-r2-storage.md | 280 ++ .../references/cloudflare-workers-advanced.md | 312 ++ .../references/cloudflare-workers-apis.md | 309 ++ .../references/cloudflare-workers-basics.md | 418 ++ skills/devops/references/docker-basics.md | 297 ++ skills/devops/references/docker-compose.md | 292 ++ skills/devops/references/gcloud-platform.md | 297 ++ skills/devops/references/gcloud-services.md | 304 ++ skills/devops/scripts/cloudflare_deploy.py | 269 + skills/devops/scripts/docker_optimize.py | 320 ++ skills/devops/scripts/requirements.txt | 20 + skills/devops/scripts/tests/requirements.txt | 3 + .../scripts/tests/test_cloudflare_deploy.py | 285 ++ .../scripts/tests/test_docker_optimize.py | 436 ++ skills/docs-seeker/SKILL.md | 207 + skills/docs-seeker/WORKFLOWS.md | 505 ++ .../docs-seeker/references/best-practices.md | 632 +++ .../references/documentation-sources.md | 461 ++ .../docs-seeker/references/error-handling.md | 621 +++ skills/docs-seeker/references/limitations.md | 821 +++ skills/docs-seeker/references/performance.md | 574 +++ .../docs-seeker/references/tool-selection.md | 262 + skills/document-skills/docx/LICENSE.txt | 30 + skills/document-skills/docx/SKILL.md | 197 + skills/document-skills/docx/docx-js.md | 350 ++ skills/document-skills/docx/ooxml.md | 610 +++ .../schemas/ISO-IEC29500-4_2016/dml-chart.xsd | 1499 ++++++ .../ISO-IEC29500-4_2016/dml-chartDrawing.xsd | 146 + .../ISO-IEC29500-4_2016/dml-diagram.xsd | 1085 ++++ .../ISO-IEC29500-4_2016/dml-lockedCanvas.xsd | 11 + .../schemas/ISO-IEC29500-4_2016/dml-main.xsd | 3081 ++++++++++++ .../ISO-IEC29500-4_2016/dml-picture.xsd | 23 + .../dml-spreadsheetDrawing.xsd | 185 + .../dml-wordprocessingDrawing.xsd | 287 ++ .../ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd | 1676 +++++++ .../shared-additionalCharacteristics.xsd | 28 + .../shared-bibliography.xsd | 144 + .../shared-commonSimpleTypes.xsd | 174 + .../shared-customXmlDataProperties.xsd | 25 + .../shared-customXmlSchemaProperties.xsd | 18 + .../shared-documentPropertiesCustom.xsd | 59 + .../shared-documentPropertiesExtended.xsd | 56 + .../shared-documentPropertiesVariantTypes.xsd | 195 + .../ISO-IEC29500-4_2016/shared-math.xsd | 582 +++ .../shared-relationshipReference.xsd | 25 + .../ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd | 4439 +++++++++++++++++ .../schemas/ISO-IEC29500-4_2016/vml-main.xsd | 570 +++ .../ISO-IEC29500-4_2016/vml-officeDrawing.xsd | 509 ++ .../vml-presentationDrawing.xsd | 12 + .../vml-spreadsheetDrawing.xsd | 108 + .../vml-wordprocessingDrawing.xsd | 96 + .../ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd | 3646 ++++++++++++++ .../ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd | 116 + .../ecma/fouth-edition/opc-contentTypes.xsd | 42 + .../ecma/fouth-edition/opc-coreProperties.xsd | 50 + .../schemas/ecma/fouth-edition/opc-digSig.xsd | 49 + .../ecma/fouth-edition/opc-relationships.xsd | 33 + .../docx/ooxml/schemas/mce/mc.xsd | 75 + .../docx/ooxml/schemas/microsoft/wml-2010.xsd | 560 +++ .../docx/ooxml/schemas/microsoft/wml-2012.xsd | 67 + .../docx/ooxml/schemas/microsoft/wml-2018.xsd | 14 + .../ooxml/schemas/microsoft/wml-cex-2018.xsd | 20 + .../ooxml/schemas/microsoft/wml-cid-2016.xsd | 13 + .../microsoft/wml-sdtdatahash-2020.xsd | 4 + .../schemas/microsoft/wml-symex-2015.xsd | 8 + .../docx/ooxml/scripts/pack.py | 159 + .../docx/ooxml/scripts/unpack.py | 29 + .../docx/ooxml/scripts/validate.py | 69 + .../docx/ooxml/scripts/validation/__init__.py | 15 + .../docx/ooxml/scripts/validation/base.py | 951 ++++ .../docx/ooxml/scripts/validation/docx.py | 274 + .../docx/ooxml/scripts/validation/pptx.py | 315 ++ .../ooxml/scripts/validation/redlining.py | 279 ++ .../document-skills/docx/scripts/__init__.py | 1 + .../document-skills/docx/scripts/document.py | 1276 +++++ .../docx/scripts/templates/comments.xml | 3 + .../scripts/templates/commentsExtended.xml | 3 + .../scripts/templates/commentsExtensible.xml | 3 + .../docx/scripts/templates/commentsIds.xml | 3 + .../docx/scripts/templates/people.xml | 3 + .../document-skills/docx/scripts/utilities.py | 374 ++ skills/document-skills/pdf/LICENSE.txt | 30 + skills/document-skills/pdf/SKILL.md | 294 ++ skills/document-skills/pdf/forms.md | 205 + skills/document-skills/pdf/reference.md | 612 +++ .../pdf/scripts/check_bounding_boxes.py | 70 + .../pdf/scripts/check_bounding_boxes_test.py | 226 + .../pdf/scripts/check_fillable_fields.py | 12 + .../pdf/scripts/convert_pdf_to_images.py | 35 + .../pdf/scripts/create_validation_image.py | 41 + .../pdf/scripts/extract_form_field_info.py | 152 + .../pdf/scripts/fill_fillable_fields.py | 114 + .../scripts/fill_pdf_form_with_annotations.py | 108 + skills/document-skills/pptx/LICENSE.txt | 30 + skills/document-skills/pptx/SKILL.md | 484 ++ skills/document-skills/pptx/html2pptx.md | 625 +++ skills/document-skills/pptx/ooxml.md | 427 ++ .../schemas/ISO-IEC29500-4_2016/dml-chart.xsd | 1499 ++++++ .../ISO-IEC29500-4_2016/dml-chartDrawing.xsd | 146 + .../ISO-IEC29500-4_2016/dml-diagram.xsd | 1085 ++++ .../ISO-IEC29500-4_2016/dml-lockedCanvas.xsd | 11 + .../schemas/ISO-IEC29500-4_2016/dml-main.xsd | 3081 ++++++++++++ .../ISO-IEC29500-4_2016/dml-picture.xsd | 23 + .../dml-spreadsheetDrawing.xsd | 185 + .../dml-wordprocessingDrawing.xsd | 287 ++ .../ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd | 1676 +++++++ .../shared-additionalCharacteristics.xsd | 28 + .../shared-bibliography.xsd | 144 + .../shared-commonSimpleTypes.xsd | 174 + .../shared-customXmlDataProperties.xsd | 25 + .../shared-customXmlSchemaProperties.xsd | 18 + .../shared-documentPropertiesCustom.xsd | 59 + .../shared-documentPropertiesExtended.xsd | 56 + .../shared-documentPropertiesVariantTypes.xsd | 195 + .../ISO-IEC29500-4_2016/shared-math.xsd | 582 +++ .../shared-relationshipReference.xsd | 25 + .../ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd | 4439 +++++++++++++++++ .../schemas/ISO-IEC29500-4_2016/vml-main.xsd | 570 +++ .../ISO-IEC29500-4_2016/vml-officeDrawing.xsd | 509 ++ .../vml-presentationDrawing.xsd | 12 + .../vml-spreadsheetDrawing.xsd | 108 + .../vml-wordprocessingDrawing.xsd | 96 + .../ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd | 3646 ++++++++++++++ .../ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd | 116 + .../ecma/fouth-edition/opc-contentTypes.xsd | 42 + .../ecma/fouth-edition/opc-coreProperties.xsd | 50 + .../schemas/ecma/fouth-edition/opc-digSig.xsd | 49 + .../ecma/fouth-edition/opc-relationships.xsd | 33 + .../pptx/ooxml/schemas/mce/mc.xsd | 75 + .../pptx/ooxml/schemas/microsoft/wml-2010.xsd | 560 +++ .../pptx/ooxml/schemas/microsoft/wml-2012.xsd | 67 + .../pptx/ooxml/schemas/microsoft/wml-2018.xsd | 14 + .../ooxml/schemas/microsoft/wml-cex-2018.xsd | 20 + .../ooxml/schemas/microsoft/wml-cid-2016.xsd | 13 + .../microsoft/wml-sdtdatahash-2020.xsd | 4 + .../schemas/microsoft/wml-symex-2015.xsd | 8 + .../pptx/ooxml/scripts/pack.py | 159 + .../pptx/ooxml/scripts/unpack.py | 29 + .../pptx/ooxml/scripts/validate.py | 69 + .../pptx/ooxml/scripts/validation/__init__.py | 15 + .../pptx/ooxml/scripts/validation/base.py | 951 ++++ .../pptx/ooxml/scripts/validation/docx.py | 274 + .../pptx/ooxml/scripts/validation/pptx.py | 315 ++ .../ooxml/scripts/validation/redlining.py | 279 ++ .../document-skills/pptx/scripts/html2pptx.js | 979 ++++ .../document-skills/pptx/scripts/inventory.py | 1020 ++++ .../document-skills/pptx/scripts/rearrange.py | 231 + .../document-skills/pptx/scripts/replace.py | 385 ++ .../document-skills/pptx/scripts/thumbnail.py | 450 ++ skills/document-skills/xlsx/LICENSE.txt | 30 + skills/document-skills/xlsx/SKILL.md | 289 ++ skills/document-skills/xlsx/recalc.py | 178 + skills/google-adk-python/SKILL.md | 237 + skills/mcp-builder/LICENSE.txt | 202 + skills/mcp-builder/SKILL.md | 328 ++ skills/mcp-builder/reference/evaluation.md | 602 +++ .../reference/mcp_best_practices.md | 915 ++++ .../mcp-builder/reference/node_mcp_server.md | 916 ++++ .../reference/python_mcp_server.md | 752 +++ skills/mcp-builder/scripts/connections.py | 151 + skills/mcp-builder/scripts/evaluation.py | 373 ++ .../scripts/example_evaluation.xml | 22 + skills/mcp-builder/scripts/requirements.txt | 2 + skills/mcp-management/README.md | 219 + skills/mcp-management/SKILL.md | 176 + skills/mcp-management/assets/tools.json | 3044 +++++++++++ .../references/configuration.md | 114 + .../references/gemini-cli-integration.md | 201 + .../mcp-management/references/mcp-protocol.md | 116 + skills/mcp-management/scripts/.env.example | 10 + skills/mcp-management/scripts/cli.ts | 155 + skills/mcp-management/scripts/mcp-client.ts | 163 + skills/mcp-management/scripts/package.json | 18 + skills/mcp-management/scripts/tsconfig.json | 15 + skills/media-processing/SKILL.md | 358 ++ .../references/ffmpeg-encoding.md | 358 ++ .../references/ffmpeg-filters.md | 503 ++ .../references/ffmpeg-streaming.md | 403 ++ .../references/format-compatibility.md | 375 ++ .../references/imagemagick-batch.md | 612 +++ .../references/imagemagick-editing.md | 623 +++ .../media-processing/scripts/batch_resize.py | 342 ++ .../media-processing/scripts/media_convert.py | 311 ++ .../media-processing/scripts/requirements.txt | 24 + .../scripts/tests/requirements.txt | 2 + .../scripts/tests/test_batch_resize.py | 372 ++ .../scripts/tests/test_media_convert.py | 259 + .../scripts/tests/test_video_optimize.py | 397 ++ .../scripts/video_optimize.py | 414 ++ skills/problem-solving/ABOUT.md | 40 + .../collision-zone-thinking/SKILL.md | 62 + .../inversion-exercise/SKILL.md | 58 + .../meta-pattern-recognition/SKILL.md | 54 + skills/problem-solving/scale-game/SKILL.md | 63 + .../simplification-cascades/SKILL.md | 76 + skills/problem-solving/when-stuck/SKILL.md | 88 + skills/repomix/SKILL.md | 215 + skills/repomix/references/configuration.md | 211 + skills/repomix/references/usage-patterns.md | 232 + skills/repomix/scripts/README.md | 179 + skills/repomix/scripts/repomix_batch.py | 455 ++ skills/repomix/scripts/repos.example.json | 15 + skills/repomix/scripts/requirements.txt | 15 + .../scripts/tests/test_repomix_batch.py | 531 ++ skills/sequential-thinking/README.md | 118 + skills/sequential-thinking/SKILL.md | 93 + .../references/advanced.md | 122 + .../references/examples.md | 274 + skills/shopify/README.md | 66 + skills/shopify/SKILL.md | 319 ++ skills/shopify/references/app-development.md | 470 ++ skills/shopify/references/extensions.md | 493 ++ skills/shopify/references/themes.md | 498 ++ skills/shopify/scripts/requirements.txt | 19 + skills/shopify/scripts/shopify_init.py | 423 ++ .../scripts/tests/test_shopify_init.py | 385 ++ skills/skill-creator/LICENSE.txt | 202 + skills/skill-creator/SKILL.md | 237 + skills/skill-creator/scripts/init_skill.py | 303 ++ skills/skill-creator/scripts/package_skill.py | 110 + .../skill-creator/scripts/quick_validate.py | 65 + skills/template-skill/SKILL.md | 6 + skills/ui-styling/LICENSE.txt | 202 + skills/ui-styling/SKILL.md | 321 ++ .../ui-styling/canvas-fonts/ArsenalSC-OFL.txt | 93 + .../canvas-fonts/ArsenalSC-Regular.ttf | Bin 0 -> 165848 bytes .../canvas-fonts/BigShoulders-Bold.ttf | Bin 0 -> 94528 bytes .../canvas-fonts/BigShoulders-OFL.txt | 93 + .../canvas-fonts/BigShoulders-Regular.ttf | Bin 0 -> 94396 bytes .../ui-styling/canvas-fonts/Boldonse-OFL.txt | 93 + .../canvas-fonts/Boldonse-Regular.ttf | Bin 0 -> 77168 bytes .../canvas-fonts/BricolageGrotesque-Bold.ttf | Bin 0 -> 90952 bytes .../canvas-fonts/BricolageGrotesque-OFL.txt | 93 + .../BricolageGrotesque-Regular.ttf | Bin 0 -> 90920 bytes .../canvas-fonts/CrimsonPro-Bold.ttf | Bin 0 -> 107352 bytes .../canvas-fonts/CrimsonPro-Italic.ttf | Bin 0 -> 108828 bytes .../canvas-fonts/CrimsonPro-OFL.txt | 93 + .../canvas-fonts/CrimsonPro-Regular.ttf | Bin 0 -> 106696 bytes skills/ui-styling/canvas-fonts/DMMono-OFL.txt | 93 + .../canvas-fonts/DMMono-Regular.ttf | Bin 0 -> 48852 bytes .../ui-styling/canvas-fonts/EricaOne-OFL.txt | 94 + .../canvas-fonts/EricaOne-Regular.ttf | Bin 0 -> 24872 bytes .../canvas-fonts/GeistMono-Bold.ttf | Bin 0 -> 78304 bytes .../ui-styling/canvas-fonts/GeistMono-OFL.txt | 93 + .../canvas-fonts/GeistMono-Regular.ttf | Bin 0 -> 78232 bytes skills/ui-styling/canvas-fonts/Gloock-OFL.txt | 93 + .../canvas-fonts/Gloock-Regular.ttf | Bin 0 -> 95156 bytes .../canvas-fonts/IBMPlexMono-Bold.ttf | Bin 0 -> 136008 bytes .../canvas-fonts/IBMPlexMono-OFL.txt | 93 + .../canvas-fonts/IBMPlexMono-Regular.ttf | Bin 0 -> 133796 bytes .../canvas-fonts/IBMPlexSerif-Bold.ttf | Bin 0 -> 161000 bytes .../canvas-fonts/IBMPlexSerif-BoldItalic.ttf | Bin 0 -> 169840 bytes .../canvas-fonts/IBMPlexSerif-Italic.ttf | Bin 0 -> 170004 bytes .../canvas-fonts/IBMPlexSerif-Regular.ttf | Bin 0 -> 160380 bytes .../canvas-fonts/InstrumentSans-Bold.ttf | Bin 0 -> 68084 bytes .../InstrumentSans-BoldItalic.ttf | Bin 0 -> 70004 bytes .../canvas-fonts/InstrumentSans-Italic.ttf | Bin 0 -> 69900 bytes .../canvas-fonts/InstrumentSans-OFL.txt | 93 + .../canvas-fonts/InstrumentSans-Regular.ttf | Bin 0 -> 68028 bytes .../canvas-fonts/InstrumentSerif-Italic.ttf | Bin 0 -> 70868 bytes .../canvas-fonts/InstrumentSerif-Regular.ttf | Bin 0 -> 69312 bytes .../ui-styling/canvas-fonts/Italiana-OFL.txt | 93 + .../canvas-fonts/Italiana-Regular.ttf | Bin 0 -> 27184 bytes .../canvas-fonts/JetBrainsMono-Bold.ttf | Bin 0 -> 114828 bytes .../canvas-fonts/JetBrainsMono-OFL.txt | 93 + .../canvas-fonts/JetBrainsMono-Regular.ttf | Bin 0 -> 114904 bytes skills/ui-styling/canvas-fonts/Jura-Light.ttf | Bin 0 -> 154308 bytes .../ui-styling/canvas-fonts/Jura-Medium.ttf | Bin 0 -> 154488 bytes skills/ui-styling/canvas-fonts/Jura-OFL.txt | 93 + .../canvas-fonts/LibreBaskerville-OFL.txt | 93 + .../canvas-fonts/LibreBaskerville-Regular.ttf | Bin 0 -> 147584 bytes skills/ui-styling/canvas-fonts/Lora-Bold.ttf | Bin 0 -> 133828 bytes .../canvas-fonts/Lora-BoldItalic.ttf | Bin 0 -> 140332 bytes .../ui-styling/canvas-fonts/Lora-Italic.ttf | Bin 0 -> 139328 bytes skills/ui-styling/canvas-fonts/Lora-OFL.txt | 93 + .../ui-styling/canvas-fonts/Lora-Regular.ttf | Bin 0 -> 133888 bytes .../canvas-fonts/NationalPark-Bold.ttf | Bin 0 -> 79208 bytes .../canvas-fonts/NationalPark-OFL.txt | 93 + .../canvas-fonts/NationalPark-Regular.ttf | Bin 0 -> 76424 bytes .../canvas-fonts/NothingYouCouldDo-OFL.txt | 93 + .../NothingYouCouldDo-Regular.ttf | Bin 0 -> 32020 bytes .../ui-styling/canvas-fonts/Outfit-Bold.ttf | Bin 0 -> 55392 bytes skills/ui-styling/canvas-fonts/Outfit-OFL.txt | 93 + .../canvas-fonts/Outfit-Regular.ttf | Bin 0 -> 54912 bytes .../canvas-fonts/PixelifySans-Medium.ttf | Bin 0 -> 51072 bytes .../canvas-fonts/PixelifySans-OFL.txt | 93 + .../ui-styling/canvas-fonts/PoiretOne-OFL.txt | 93 + .../canvas-fonts/PoiretOne-Regular.ttf | Bin 0 -> 45244 bytes .../canvas-fonts/RedHatMono-Bold.ttf | Bin 0 -> 34420 bytes .../canvas-fonts/RedHatMono-OFL.txt | 93 + .../canvas-fonts/RedHatMono-Regular.ttf | Bin 0 -> 34488 bytes .../canvas-fonts/Silkscreen-OFL.txt | 93 + .../canvas-fonts/Silkscreen-Regular.ttf | Bin 0 -> 31960 bytes .../canvas-fonts/SmoochSans-Medium.ttf | Bin 0 -> 59704 bytes .../canvas-fonts/SmoochSans-OFL.txt | 93 + .../ui-styling/canvas-fonts/Tektur-Medium.ttf | Bin 0 -> 76248 bytes skills/ui-styling/canvas-fonts/Tektur-OFL.txt | 93 + .../canvas-fonts/Tektur-Regular.ttf | Bin 0 -> 75604 bytes .../ui-styling/canvas-fonts/WorkSans-Bold.ttf | Bin 0 -> 191304 bytes .../canvas-fonts/WorkSans-BoldItalic.ttf | Bin 0 -> 175772 bytes .../canvas-fonts/WorkSans-Italic.ttf | Bin 0 -> 174280 bytes .../ui-styling/canvas-fonts/WorkSans-OFL.txt | 93 + .../canvas-fonts/WorkSans-Regular.ttf | Bin 0 -> 188916 bytes .../canvas-fonts/YoungSerif-OFL.txt | 93 + .../canvas-fonts/YoungSerif-Regular.ttf | Bin 0 -> 105136 bytes .../references/canvas-design-system.md | 320 ++ .../references/shadcn-accessibility.md | 471 ++ .../references/shadcn-components.md | 424 ++ .../ui-styling/references/shadcn-theming.md | 373 ++ .../references/tailwind-customization.md | 483 ++ .../references/tailwind-responsive.md | 382 ++ .../references/tailwind-utilities.md | 455 ++ skills/ui-styling/scripts/requirements.txt | 17 + skills/ui-styling/scripts/shadcn_add.py | 292 ++ .../ui-styling/scripts/tailwind_config_gen.py | 456 ++ .../ui-styling/scripts/tests/coverage-ui.json | 1 + .../ui-styling/scripts/tests/requirements.txt | 3 + .../scripts/tests/test_shadcn_add.py | 266 + .../scripts/tests/test_tailwind_config_gen.py | 336 ++ skills/web-frameworks/SKILL.md | 324 ++ .../references/nextjs-app-router.md | 465 ++ .../references/nextjs-data-fetching.md | 459 ++ .../references/nextjs-optimization.md | 511 ++ .../references/nextjs-server-components.md | 495 ++ .../references/remix-icon-integration.md | 603 +++ .../references/turborepo-caching.md | 551 ++ .../references/turborepo-pipelines.md | 517 ++ .../references/turborepo-setup.md | 542 ++ skills/web-frameworks/scripts/__init__.py | 0 skills/web-frameworks/scripts/nextjs_init.py | 547 ++ .../web-frameworks/scripts/requirements.txt | 16 + .../scripts/tests/coverage-web.json | 1 + .../scripts/tests/requirements.txt | 3 + .../scripts/tests/test_nextjs_init.py | 319 ++ .../scripts/tests/test_turborepo_migrate.py | 374 ++ .../scripts/turborepo_migrate.py | 394 ++ 434 files changed, 125248 insertions(+) create mode 100644 .claude-plugin/plugin.json create mode 100644 README.md create mode 100644 plugin.lock.json create mode 100644 skills/ai-multimodal/.env.example create mode 100644 skills/ai-multimodal/SKILL.md create mode 100644 skills/ai-multimodal/references/audio-processing.md create mode 100644 skills/ai-multimodal/references/image-generation.md create mode 100644 skills/ai-multimodal/references/video-analysis.md create mode 100644 skills/ai-multimodal/references/vision-understanding.md create mode 100644 skills/ai-multimodal/scripts/document_converter.py create mode 100644 skills/ai-multimodal/scripts/gemini_batch_process.py create mode 100644 skills/ai-multimodal/scripts/media_optimizer.py create mode 100644 skills/ai-multimodal/scripts/requirements.txt create mode 100644 skills/ai-multimodal/scripts/tests/requirements.txt create mode 100644 skills/ai-multimodal/scripts/tests/test_document_converter.py create mode 100644 skills/ai-multimodal/scripts/tests/test_gemini_batch_process.py create mode 100644 skills/ai-multimodal/scripts/tests/test_media_optimizer.py create mode 100644 skills/better-auth/SKILL.md create mode 100644 skills/better-auth/references/advanced-features.md create mode 100644 skills/better-auth/references/database-integration.md create mode 100644 skills/better-auth/references/email-password-auth.md create mode 100644 skills/better-auth/references/oauth-providers.md create mode 100644 skills/better-auth/scripts/better_auth_init.py create mode 100644 skills/better-auth/scripts/requirements.txt create mode 100644 skills/better-auth/scripts/tests/test_better_auth_init.py create mode 100644 skills/chrome-devtools/SKILL.md create mode 100644 skills/chrome-devtools/references/cdp-domains.md create mode 100644 skills/chrome-devtools/references/performance-guide.md create mode 100644 skills/chrome-devtools/references/puppeteer-reference.md create mode 100644 skills/chrome-devtools/scripts/README.md create mode 100644 skills/chrome-devtools/scripts/__tests__/selector.test.js create mode 100644 skills/chrome-devtools/scripts/click.js create mode 100644 skills/chrome-devtools/scripts/console.js create mode 100644 skills/chrome-devtools/scripts/evaluate.js create mode 100644 skills/chrome-devtools/scripts/fill.js create mode 100644 skills/chrome-devtools/scripts/install-deps.sh create mode 100644 skills/chrome-devtools/scripts/install.sh create mode 100644 skills/chrome-devtools/scripts/navigate.js create mode 100644 skills/chrome-devtools/scripts/network.js create mode 100644 skills/chrome-devtools/scripts/package.json create mode 100644 skills/chrome-devtools/scripts/performance.js create mode 100644 skills/chrome-devtools/scripts/screenshot.js create mode 100644 skills/chrome-devtools/scripts/snapshot.js create mode 100644 skills/claude-code/SKILL.md create mode 100644 skills/claude-code/llms.txt create mode 100644 skills/claude-code/references/advanced-features.md create mode 100644 skills/claude-code/references/agent-skills.md create mode 100644 skills/claude-code/references/api-reference.md create mode 100644 skills/claude-code/references/best-practices.md create mode 100644 skills/claude-code/references/cicd-integration.md create mode 100644 skills/claude-code/references/configuration.md create mode 100644 skills/claude-code/references/enterprise-features.md create mode 100644 skills/claude-code/references/getting-started.md create mode 100644 skills/claude-code/references/hooks-and-plugins.md create mode 100644 skills/claude-code/references/ide-integration.md create mode 100644 skills/claude-code/references/mcp-integration.md create mode 100644 skills/claude-code/references/slash-commands.md create mode 100644 skills/claude-code/references/troubleshooting.md create mode 100644 skills/claude-code/skill.json create mode 100644 skills/code-review/SKILL.md create mode 100644 skills/code-review/references/code-review-reception.md create mode 100644 skills/code-review/references/requesting-code-review.md create mode 100644 skills/code-review/references/verification-before-completion.md create mode 100644 skills/common/README.md create mode 100644 skills/common/api_key_helper.py create mode 100644 skills/databases/SKILL.md create mode 100644 skills/databases/references/mongodb-aggregation.md create mode 100644 skills/databases/references/mongodb-atlas.md create mode 100644 skills/databases/references/mongodb-crud.md create mode 100644 skills/databases/references/mongodb-indexing.md create mode 100644 skills/databases/references/postgresql-administration.md create mode 100644 skills/databases/references/postgresql-performance.md create mode 100644 skills/databases/references/postgresql-psql-cli.md create mode 100644 skills/databases/references/postgresql-queries.md create mode 100644 skills/databases/scripts/db_backup.py create mode 100644 skills/databases/scripts/db_migrate.py create mode 100644 skills/databases/scripts/db_performance_check.py create mode 100644 skills/databases/scripts/requirements.txt create mode 100644 skills/databases/scripts/tests/coverage-db.json create mode 100644 skills/databases/scripts/tests/requirements.txt create mode 100644 skills/databases/scripts/tests/test_db_backup.py create mode 100644 skills/databases/scripts/tests/test_db_migrate.py create mode 100644 skills/databases/scripts/tests/test_db_performance_check.py create mode 100644 skills/debugging/defense-in-depth/SKILL.md create mode 100644 skills/debugging/root-cause-tracing/SKILL.md create mode 100755 skills/debugging/root-cause-tracing/find-polluter.sh create mode 100644 skills/debugging/systematic-debugging/CREATION-LOG.md create mode 100644 skills/debugging/systematic-debugging/SKILL.md create mode 100644 skills/debugging/systematic-debugging/test-academic.md create mode 100644 skills/debugging/systematic-debugging/test-pressure-1.md create mode 100644 skills/debugging/systematic-debugging/test-pressure-2.md create mode 100644 skills/debugging/systematic-debugging/test-pressure-3.md create mode 100644 skills/debugging/verification-before-completion/SKILL.md create mode 100644 skills/devops/.env.example create mode 100644 skills/devops/SKILL.md create mode 100644 skills/devops/references/browser-rendering.md create mode 100644 skills/devops/references/cloudflare-d1-kv.md create mode 100644 skills/devops/references/cloudflare-platform.md create mode 100644 skills/devops/references/cloudflare-r2-storage.md create mode 100644 skills/devops/references/cloudflare-workers-advanced.md create mode 100644 skills/devops/references/cloudflare-workers-apis.md create mode 100644 skills/devops/references/cloudflare-workers-basics.md create mode 100644 skills/devops/references/docker-basics.md create mode 100644 skills/devops/references/docker-compose.md create mode 100644 skills/devops/references/gcloud-platform.md create mode 100644 skills/devops/references/gcloud-services.md create mode 100644 skills/devops/scripts/cloudflare_deploy.py create mode 100644 skills/devops/scripts/docker_optimize.py create mode 100644 skills/devops/scripts/requirements.txt create mode 100644 skills/devops/scripts/tests/requirements.txt create mode 100644 skills/devops/scripts/tests/test_cloudflare_deploy.py create mode 100644 skills/devops/scripts/tests/test_docker_optimize.py create mode 100644 skills/docs-seeker/SKILL.md create mode 100644 skills/docs-seeker/WORKFLOWS.md create mode 100644 skills/docs-seeker/references/best-practices.md create mode 100644 skills/docs-seeker/references/documentation-sources.md create mode 100644 skills/docs-seeker/references/error-handling.md create mode 100644 skills/docs-seeker/references/limitations.md create mode 100644 skills/docs-seeker/references/performance.md create mode 100644 skills/docs-seeker/references/tool-selection.md create mode 100644 skills/document-skills/docx/LICENSE.txt create mode 100644 skills/document-skills/docx/SKILL.md create mode 100644 skills/document-skills/docx/docx-js.md create mode 100644 skills/document-skills/docx/ooxml.md create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/mce/mc.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-2010.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-2012.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-2018.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-cex-2018.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-cid-2016.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd create mode 100644 skills/document-skills/docx/ooxml/schemas/microsoft/wml-symex-2015.xsd create mode 100755 skills/document-skills/docx/ooxml/scripts/pack.py create mode 100755 skills/document-skills/docx/ooxml/scripts/unpack.py create mode 100755 skills/document-skills/docx/ooxml/scripts/validate.py create mode 100644 skills/document-skills/docx/ooxml/scripts/validation/__init__.py create mode 100644 skills/document-skills/docx/ooxml/scripts/validation/base.py create mode 100644 skills/document-skills/docx/ooxml/scripts/validation/docx.py create mode 100644 skills/document-skills/docx/ooxml/scripts/validation/pptx.py create mode 100644 skills/document-skills/docx/ooxml/scripts/validation/redlining.py create mode 100755 skills/document-skills/docx/scripts/__init__.py create mode 100755 skills/document-skills/docx/scripts/document.py create mode 100644 skills/document-skills/docx/scripts/templates/comments.xml create mode 100644 skills/document-skills/docx/scripts/templates/commentsExtended.xml create mode 100644 skills/document-skills/docx/scripts/templates/commentsExtensible.xml create mode 100644 skills/document-skills/docx/scripts/templates/commentsIds.xml create mode 100644 skills/document-skills/docx/scripts/templates/people.xml create mode 100755 skills/document-skills/docx/scripts/utilities.py create mode 100644 skills/document-skills/pdf/LICENSE.txt create mode 100644 skills/document-skills/pdf/SKILL.md create mode 100644 skills/document-skills/pdf/forms.md create mode 100644 skills/document-skills/pdf/reference.md create mode 100644 skills/document-skills/pdf/scripts/check_bounding_boxes.py create mode 100644 skills/document-skills/pdf/scripts/check_bounding_boxes_test.py create mode 100644 skills/document-skills/pdf/scripts/check_fillable_fields.py create mode 100644 skills/document-skills/pdf/scripts/convert_pdf_to_images.py create mode 100644 skills/document-skills/pdf/scripts/create_validation_image.py create mode 100644 skills/document-skills/pdf/scripts/extract_form_field_info.py create mode 100644 skills/document-skills/pdf/scripts/fill_fillable_fields.py create mode 100644 skills/document-skills/pdf/scripts/fill_pdf_form_with_annotations.py create mode 100644 skills/document-skills/pptx/LICENSE.txt create mode 100644 skills/document-skills/pptx/SKILL.md create mode 100644 skills/document-skills/pptx/html2pptx.md create mode 100644 skills/document-skills/pptx/ooxml.md create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/mce/mc.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-2010.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-2012.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-2018.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-cex-2018.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-cid-2016.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd create mode 100644 skills/document-skills/pptx/ooxml/schemas/microsoft/wml-symex-2015.xsd create mode 100755 skills/document-skills/pptx/ooxml/scripts/pack.py create mode 100755 skills/document-skills/pptx/ooxml/scripts/unpack.py create mode 100755 skills/document-skills/pptx/ooxml/scripts/validate.py create mode 100644 skills/document-skills/pptx/ooxml/scripts/validation/__init__.py create mode 100644 skills/document-skills/pptx/ooxml/scripts/validation/base.py create mode 100644 skills/document-skills/pptx/ooxml/scripts/validation/docx.py create mode 100644 skills/document-skills/pptx/ooxml/scripts/validation/pptx.py create mode 100644 skills/document-skills/pptx/ooxml/scripts/validation/redlining.py create mode 100755 skills/document-skills/pptx/scripts/html2pptx.js create mode 100755 skills/document-skills/pptx/scripts/inventory.py create mode 100755 skills/document-skills/pptx/scripts/rearrange.py create mode 100755 skills/document-skills/pptx/scripts/replace.py create mode 100755 skills/document-skills/pptx/scripts/thumbnail.py create mode 100644 skills/document-skills/xlsx/LICENSE.txt create mode 100644 skills/document-skills/xlsx/SKILL.md create mode 100644 skills/document-skills/xlsx/recalc.py create mode 100644 skills/google-adk-python/SKILL.md create mode 100644 skills/mcp-builder/LICENSE.txt create mode 100644 skills/mcp-builder/SKILL.md create mode 100644 skills/mcp-builder/reference/evaluation.md create mode 100644 skills/mcp-builder/reference/mcp_best_practices.md create mode 100644 skills/mcp-builder/reference/node_mcp_server.md create mode 100644 skills/mcp-builder/reference/python_mcp_server.md create mode 100644 skills/mcp-builder/scripts/connections.py create mode 100644 skills/mcp-builder/scripts/evaluation.py create mode 100644 skills/mcp-builder/scripts/example_evaluation.xml create mode 100644 skills/mcp-builder/scripts/requirements.txt create mode 100644 skills/mcp-management/README.md create mode 100644 skills/mcp-management/SKILL.md create mode 100644 skills/mcp-management/assets/tools.json create mode 100644 skills/mcp-management/references/configuration.md create mode 100644 skills/mcp-management/references/gemini-cli-integration.md create mode 100644 skills/mcp-management/references/mcp-protocol.md create mode 100644 skills/mcp-management/scripts/.env.example create mode 100644 skills/mcp-management/scripts/cli.ts create mode 100644 skills/mcp-management/scripts/mcp-client.ts create mode 100644 skills/mcp-management/scripts/package.json create mode 100644 skills/mcp-management/scripts/tsconfig.json create mode 100644 skills/media-processing/SKILL.md create mode 100644 skills/media-processing/references/ffmpeg-encoding.md create mode 100644 skills/media-processing/references/ffmpeg-filters.md create mode 100644 skills/media-processing/references/ffmpeg-streaming.md create mode 100644 skills/media-processing/references/format-compatibility.md create mode 100644 skills/media-processing/references/imagemagick-batch.md create mode 100644 skills/media-processing/references/imagemagick-editing.md create mode 100644 skills/media-processing/scripts/batch_resize.py create mode 100644 skills/media-processing/scripts/media_convert.py create mode 100644 skills/media-processing/scripts/requirements.txt create mode 100644 skills/media-processing/scripts/tests/requirements.txt create mode 100644 skills/media-processing/scripts/tests/test_batch_resize.py create mode 100644 skills/media-processing/scripts/tests/test_media_convert.py create mode 100644 skills/media-processing/scripts/tests/test_video_optimize.py create mode 100644 skills/media-processing/scripts/video_optimize.py create mode 100644 skills/problem-solving/ABOUT.md create mode 100644 skills/problem-solving/collision-zone-thinking/SKILL.md create mode 100644 skills/problem-solving/inversion-exercise/SKILL.md create mode 100644 skills/problem-solving/meta-pattern-recognition/SKILL.md create mode 100644 skills/problem-solving/scale-game/SKILL.md create mode 100644 skills/problem-solving/simplification-cascades/SKILL.md create mode 100644 skills/problem-solving/when-stuck/SKILL.md create mode 100644 skills/repomix/SKILL.md create mode 100644 skills/repomix/references/configuration.md create mode 100644 skills/repomix/references/usage-patterns.md create mode 100644 skills/repomix/scripts/README.md create mode 100644 skills/repomix/scripts/repomix_batch.py create mode 100644 skills/repomix/scripts/repos.example.json create mode 100644 skills/repomix/scripts/requirements.txt create mode 100644 skills/repomix/scripts/tests/test_repomix_batch.py create mode 100644 skills/sequential-thinking/README.md create mode 100644 skills/sequential-thinking/SKILL.md create mode 100644 skills/sequential-thinking/references/advanced.md create mode 100644 skills/sequential-thinking/references/examples.md create mode 100644 skills/shopify/README.md create mode 100644 skills/shopify/SKILL.md create mode 100644 skills/shopify/references/app-development.md create mode 100644 skills/shopify/references/extensions.md create mode 100644 skills/shopify/references/themes.md create mode 100644 skills/shopify/scripts/requirements.txt create mode 100644 skills/shopify/scripts/shopify_init.py create mode 100644 skills/shopify/scripts/tests/test_shopify_init.py create mode 100644 skills/skill-creator/LICENSE.txt create mode 100644 skills/skill-creator/SKILL.md create mode 100755 skills/skill-creator/scripts/init_skill.py create mode 100755 skills/skill-creator/scripts/package_skill.py create mode 100755 skills/skill-creator/scripts/quick_validate.py create mode 100644 skills/template-skill/SKILL.md create mode 100644 skills/ui-styling/LICENSE.txt create mode 100644 skills/ui-styling/SKILL.md create mode 100644 skills/ui-styling/canvas-fonts/ArsenalSC-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/ArsenalSC-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/BigShoulders-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/BigShoulders-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/BigShoulders-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Boldonse-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Boldonse-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/BricolageGrotesque-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/BricolageGrotesque-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/BricolageGrotesque-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/CrimsonPro-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/CrimsonPro-Italic.ttf create mode 100644 skills/ui-styling/canvas-fonts/CrimsonPro-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/CrimsonPro-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/DMMono-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/DMMono-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/EricaOne-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/EricaOne-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/GeistMono-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/GeistMono-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/GeistMono-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Gloock-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Gloock-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexMono-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexMono-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexMono-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexSerif-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexSerif-BoldItalic.ttf create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexSerif-Italic.ttf create mode 100644 skills/ui-styling/canvas-fonts/IBMPlexSerif-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSans-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSans-BoldItalic.ttf create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSans-Italic.ttf create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSans-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSans-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSerif-Italic.ttf create mode 100644 skills/ui-styling/canvas-fonts/InstrumentSerif-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Italiana-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Italiana-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/JetBrainsMono-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/JetBrainsMono-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/JetBrainsMono-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Jura-Light.ttf create mode 100644 skills/ui-styling/canvas-fonts/Jura-Medium.ttf create mode 100644 skills/ui-styling/canvas-fonts/Jura-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/LibreBaskerville-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/LibreBaskerville-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Lora-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/Lora-BoldItalic.ttf create mode 100644 skills/ui-styling/canvas-fonts/Lora-Italic.ttf create mode 100644 skills/ui-styling/canvas-fonts/Lora-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Lora-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/NationalPark-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/NationalPark-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/NationalPark-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/NothingYouCouldDo-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/NothingYouCouldDo-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Outfit-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/Outfit-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Outfit-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/PixelifySans-Medium.ttf create mode 100644 skills/ui-styling/canvas-fonts/PixelifySans-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/PoiretOne-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/PoiretOne-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/RedHatMono-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/RedHatMono-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/RedHatMono-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/Silkscreen-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Silkscreen-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/SmoochSans-Medium.ttf create mode 100644 skills/ui-styling/canvas-fonts/SmoochSans-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Tektur-Medium.ttf create mode 100644 skills/ui-styling/canvas-fonts/Tektur-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/Tektur-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/WorkSans-Bold.ttf create mode 100644 skills/ui-styling/canvas-fonts/WorkSans-BoldItalic.ttf create mode 100644 skills/ui-styling/canvas-fonts/WorkSans-Italic.ttf create mode 100644 skills/ui-styling/canvas-fonts/WorkSans-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/WorkSans-Regular.ttf create mode 100644 skills/ui-styling/canvas-fonts/YoungSerif-OFL.txt create mode 100644 skills/ui-styling/canvas-fonts/YoungSerif-Regular.ttf create mode 100644 skills/ui-styling/references/canvas-design-system.md create mode 100644 skills/ui-styling/references/shadcn-accessibility.md create mode 100644 skills/ui-styling/references/shadcn-components.md create mode 100644 skills/ui-styling/references/shadcn-theming.md create mode 100644 skills/ui-styling/references/tailwind-customization.md create mode 100644 skills/ui-styling/references/tailwind-responsive.md create mode 100644 skills/ui-styling/references/tailwind-utilities.md create mode 100644 skills/ui-styling/scripts/requirements.txt create mode 100644 skills/ui-styling/scripts/shadcn_add.py create mode 100644 skills/ui-styling/scripts/tailwind_config_gen.py create mode 100644 skills/ui-styling/scripts/tests/coverage-ui.json create mode 100644 skills/ui-styling/scripts/tests/requirements.txt create mode 100644 skills/ui-styling/scripts/tests/test_shadcn_add.py create mode 100644 skills/ui-styling/scripts/tests/test_tailwind_config_gen.py create mode 100644 skills/web-frameworks/SKILL.md create mode 100644 skills/web-frameworks/references/nextjs-app-router.md create mode 100644 skills/web-frameworks/references/nextjs-data-fetching.md create mode 100644 skills/web-frameworks/references/nextjs-optimization.md create mode 100644 skills/web-frameworks/references/nextjs-server-components.md create mode 100644 skills/web-frameworks/references/remix-icon-integration.md create mode 100644 skills/web-frameworks/references/turborepo-caching.md create mode 100644 skills/web-frameworks/references/turborepo-pipelines.md create mode 100644 skills/web-frameworks/references/turborepo-setup.md create mode 100644 skills/web-frameworks/scripts/__init__.py create mode 100644 skills/web-frameworks/scripts/nextjs_init.py create mode 100644 skills/web-frameworks/scripts/requirements.txt create mode 100644 skills/web-frameworks/scripts/tests/coverage-web.json create mode 100644 skills/web-frameworks/scripts/tests/requirements.txt create mode 100644 skills/web-frameworks/scripts/tests/test_nextjs_init.py create mode 100644 skills/web-frameworks/scripts/tests/test_turborepo_migrate.py create mode 100644 skills/web-frameworks/scripts/turborepo_migrate.py diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..877d7b1 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,34 @@ +{ + "name": "claudekit-skills", + "description": "ClaudeKit Skills - Comprehensive collection of specialized agent skills, commands, and agents for authentication, AI/ML, web development, cloud platforms, databases, debugging, documentation, problem-solving, and more", + "version": "main", + "author": { + "name": "mrgoonie", + "url": "https://github.com/mrgoonie" + }, + "skills": [ + "./skills/ai-multimodal", + "./skills/better-auth", + "./skills/chrome-devtools", + "./skills/claude-code", + "./skills/code-review", + "./skills/common", + "./skills/databases", + "./skills/debugging", + "./skills/devops", + "./skills/docs-seeker", + "./skills/document-skills", + "./skills/google-adk-python", + "./skills/mcp-builder", + "./skills/mcp-management", + "./skills/media-processing", + "./skills/problem-solving", + "./skills/repomix", + "./skills/sequential-thinking", + "./skills/shopify", + "./skills/skill-creator", + "./skills/template-skill", + "./skills/ui-styling", + "./skills/web-frameworks" + ] +} \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..6cd9f7b --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# claudekit-skills + +ClaudeKit Skills - Comprehensive collection of specialized agent skills, commands, and agents for authentication, AI/ML, web development, cloud platforms, databases, debugging, documentation, problem-solving, and more diff --git a/plugin.lock.json b/plugin.lock.json new file mode 100644 index 0000000..c539038 --- /dev/null +++ b/plugin.lock.json @@ -0,0 +1,1813 @@ +{ + "$schema": "internal://schemas/plugin.lock.v1.json", + "pluginId": "gh:rafaelcalleja/claude-market-place:plugins/claudekit-skills", + "normalized": { + "repo": null, + "ref": "refs/tags/v20251128.0", + "commit": "fc08264fe92f3b17320b08ef5bd9ba2eb5808228", + "treeHash": "67012aff99a0fb8498e5a2234fd4ab558e85c60d85822d5ab8a35ad610581bf2", + "generatedAt": "2025-11-28T10:27:45.889460Z", + "toolVersion": "publish_plugins.py@0.2.0" + }, + "origin": { + "remote": "git@github.com:zhongweili/42plugin-data.git", + "branch": "master", + "commit": "aa1497ed0949fd50e99e70d6324a29c5b34f9390", + "repoRoot": "/Users/zhongweili/projects/openmind/42plugin-data" + }, + "manifest": { + "name": "claudekit-skills", + "description": "ClaudeKit Skills - Comprehensive collection of specialized agent skills, commands, and agents for authentication, AI/ML, web development, cloud platforms, databases, debugging, documentation, problem-solving, and more", + "version": "main" + }, + "content": { + "files": [ + { + "path": "README.md", + "sha256": "8de63066125e7a13e6938373eedd546e494591262b07c5756d62030187f45aa2" + }, + { + "path": ".claude-plugin/plugin.json", + "sha256": "c7368b180604e604b9094fdf93f3e0c562d724eb4fbd56d0f299532b5ec29a59" + }, + { + "path": "skills/document-skills/xlsx/recalc.py", + "sha256": "ab1ef0c94536bb23b6c6a3d32769b0401ec3cc85e73c247d574dd84ec73af15d" + }, + { + "path": "skills/document-skills/xlsx/SKILL.md", + "sha256": "020ccdb5932257b66c638ec1157ea248d57fa52c8c01f1f68b559b5970c7df35" + }, + { + "path": "skills/document-skills/xlsx/LICENSE.txt", + "sha256": "79f6d8f5b427252fa3b1c11ecdbdb6bf610b944f7530b4de78f770f38741cfaa" + }, + { + "path": "skills/document-skills/pdf/reference.md", + "sha256": "03a5f964f8abecbbe156f363356e927e864d7ee964f1012c84ee1bfc8acbeb95" + }, + { + "path": "skills/document-skills/pdf/forms.md", + "sha256": "0ab10e9095deb1c1f9f79eb04254589f55c1d16e095cb53191e03f9fc3184449" + }, + { + "path": "skills/document-skills/pdf/SKILL.md", + "sha256": "38d8559d4899602f82f3560052132aa0c40cfca80203037729756bdb4fb8e0cb" + }, + { + "path": "skills/document-skills/pdf/LICENSE.txt", + "sha256": "79f6d8f5b427252fa3b1c11ecdbdb6bf610b944f7530b4de78f770f38741cfaa" + }, + { + "path": "skills/document-skills/pdf/scripts/fill_fillable_fields.py", + "sha256": "65b3e41969707022283a313a4cf9696d31793cbe255dffe13370e75abda448a7" + }, + { + "path": "skills/document-skills/pdf/scripts/convert_pdf_to_images.py", + "sha256": "095a0105a718af75ede309cb03f84a20c81d17f1727f7686fd4b294f1f40294f" + }, + { + "path": "skills/document-skills/pdf/scripts/extract_form_field_info.py", + "sha256": "9db1a2720cf54223cdc4bf797080c70f4e0d27288d9f400e066c14524519021d" + }, + { + "path": "skills/document-skills/pdf/scripts/check_bounding_boxes.py", + "sha256": "eb2a5f79c8aa10c57b5867e1f0fc75b52a68b1218442ef9d838dfb4b9eedc6f4" + }, + { + "path": "skills/document-skills/pdf/scripts/check_bounding_boxes_test.py", + "sha256": "f95dca01a8b79aafd152511e9f7bf2bbcd606dde1be77d691f03a18624e002ca" + }, + { + "path": "skills/document-skills/pdf/scripts/create_validation_image.py", + "sha256": "89675be66b48925d7b498eb9454521c78cf9e9ff188ebf094934b598550effe5" + }, + { + "path": "skills/document-skills/pdf/scripts/fill_pdf_form_with_annotations.py", + "sha256": "599d6f307edb4ee6b837f21d0ea860c41c22246e270b45d6bc750c5b87c86ce0" + }, + { + "path": "skills/document-skills/pdf/scripts/check_fillable_fields.py", + "sha256": "250d5aa4e8451d6a83d17d3550c14e6c844ac347145f916ebf7980b118312b41" + }, + { + "path": "skills/document-skills/pptx/ooxml.md", + "sha256": "09868e9f1786765421ecf3f0f49c77006738efda82a76df43ed87f7a9bfe2467" + }, + { + "path": "skills/document-skills/pptx/SKILL.md", + "sha256": "b6f25545bfb358739f1532f793458b5dbc87ee009933cb7c306b2d951ab6617f" + }, + { + "path": "skills/document-skills/pptx/html2pptx.md", + "sha256": "f08ed7580969b796d9cd5ade93e2cdee981dcaf13cc5eb12e8d4a3700c2d6047" + }, + { + "path": "skills/document-skills/pptx/LICENSE.txt", + "sha256": "79f6d8f5b427252fa3b1c11ecdbdb6bf610b944f7530b4de78f770f38741cfaa" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd", + "sha256": "842e7163409c8d74f4d7088a8bc99500d80bc75332681a0980055b08f374a604" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-2012.xsd", + "sha256": "0fa75578a000439a7988ba0c59fdc69f774bbd416cbacc14d07125b3f686cb74" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-2010.xsd", + "sha256": "568b26ee156cb9549aa439ca2158965f77b7c1602b7e0316f40ac6cf586e35f2" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-cid-2016.xsd", + "sha256": "127ca209fa73d7cb708449cb355c871867948a96e4a74f7bf5811ef62d17991d" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-symex-2015.xsd", + "sha256": "16f6f8072249f431370723c2cd8974672e0d9c897e00e97dd918079df934871b" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-cex-2018.xsd", + "sha256": "fddc2b880cabb9005aebbc7e783e53c19fec1c03df7d0e2f2076a33a0fdfd081" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/microsoft/wml-2018.xsd", + "sha256": "be0ff793a22dd31384650c3a4da14c2fa8062751c2e97b0e5ee852bda13c60ad" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/mce/mc.xsd", + "sha256": "3a37e461ecf5a8670fdec34029703401f8728ab9c96ec1739a6ae58d55212413" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd", + "sha256": "451958454e8588dfc7cd945981ada142ca06ff3307937f5700df059c2b307fa8" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd", + "sha256": "f565adfef5a502044abc3a9153e157edc25af78304d335994afb958874b15e26" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd", + "sha256": "9e0b7209fc69ab11987900404540969976000c5ebe4d4f58c43dc3842886bf3a" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd", + "sha256": "6de111e11403f7cd49027400755bae0ea1cabef2815f09bd40a24f0017613b24" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd", + "sha256": "133c9f64a5c5d573b78d0a474122b22506d8eadb5e063f67cdbbb8fa2f161d0e" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd", + "sha256": "585bedc1313b40888dcc544cb74cd939a105ee674f3b1d3aa1cc6d34f70ff155" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd", + "sha256": "0d103b99a4a8652f8871552a69d42d2a3760ac6a5e3ef02d979c4273257ff6a4" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd", + "sha256": "d173c3e5d61e42e2e3a97226c632fd2ab7cc481fc4e492365b87024ab546daff" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd", + "sha256": "5cb76dabd8b97d1e9308a1700b90c20139be4d50792d21a7f09789f5cccd6026" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd", + "sha256": "41b93bd8857cc68b1e43be2806a872d736a9bdd6566900062d8fdb57d7bbb354" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd", + "sha256": "3fd0586f2637b98bb9886f0e0b67d89e1cc987c2d158cc7deb5f5b9890ced412" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd", + "sha256": "beffeed56945c22a77440122c8bdc426f3fcbe7f3b12ea0976c770d1f8d54578" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd", + "sha256": "6bdeb169c3717eb01108853bd9fc5a3750fb1fa5b82abbdd854d49855a40f519" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd", + "sha256": "c2dd9f61f892deae6acd8d20771ea79b12018af25f3bf8d06639c8542d218cfd" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd", + "sha256": "5d389d42befbebd91945d620242347caecd3367f9a3a7cf8d97949507ae1f53c" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd", + "sha256": "29b254ee0d10414a8504b5a08149c7baec35a60d5ff607d6b3f492aa36815f40" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd", + "sha256": "5375417f0f5394b8dd1a7035b9679151f19a6b65df309dec10cfb4a420cb00e9" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd", + "sha256": "9c085407751b9061c1f996f6c39ce58451be22a8d334f09175f0e89e42736285" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd", + "sha256": "b4532b6d258832953fbb3ee4c711f4fe25d3faf46a10644b2505f17010d01e88" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd", + "sha256": "e2abacbb9a55ce1365f8961bc1b1395bbc811e512b111000d8c333f98458dece" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd", + "sha256": "bdad416b096b61d37b71603b2c949484f9070c830bdaeba93bf35e15c8900614" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd", + "sha256": "475dcae1e7d1ea46232db6f8481040c15e53a52a3c256831d3df204212b0e831" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd", + "sha256": "0b364451dc36a48dd6dae0f3b6ada05fd9b71e5208211f8ee5537d7e51a587e2" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd", + "sha256": "bc92e36ccd233722d4c5869bec71ddc7b12e2df56059942cce5a39065cc9c368" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd", + "sha256": "f5ee623b08b6a66935e5aced2f5d8ad0fc71bf9e8e833cd490150c0fa94b8763" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd", + "sha256": "a539aa2fb154fa50e0f5cc97e6ad7cbc66f8ec3e3746f61ec6a8b0d5d15ecdf2" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd", + "sha256": "12264f3c03d738311cd9237d212f1c07479e70f0cbe1ae725d29b36539aef637" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd", + "sha256": "0ef4bb354ff44b923564c4ddbdda5987919d220225129ec94614a618ceafc281" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd", + "sha256": "7b5b7413e2c895b1e148e82e292a117d53c7ec65b0696c992edca57b61b4a74b" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd", + "sha256": "3213ef1631606250f5010b42cad7ef716f7c59426367798e33c374c0ec391d3a" + }, + { + "path": "skills/document-skills/pptx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd", + "sha256": "3c6709101c6aaa82888df5d8795c33f9e857196790eb320d9194e64be2b6bdd8" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/pack.py", + "sha256": "6fe762f45aff8c63fd95b9fcb1337b28921d6fa454e18a0e8158d4c8708d6d00" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/validate.py", + "sha256": "1ec252de8b14b07d16966c48906ccb1c45c68bcd23557ad31d8c50a27f5f8c0f" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/unpack.py", + "sha256": "0bd17f76a1a4c388aba42c6d1d39015fa84e405c3e0692397fe12762bd632b58" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/validation/docx.py", + "sha256": "e65d6cda0525866a24cc847b2e883bd2416ae6f87b3f5b9e2784dfbb0ec13093" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/validation/__init__.py", + "sha256": "83e0f035c5abea238d3f2c3968afbd511ed022b527b7c9cb60a9434cc34ff987" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/validation/redlining.py", + "sha256": "97abfdff4f08f43f9a4bb5c8a2f8fd483398b5b339592724e8635153b5507967" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/validation/pptx.py", + "sha256": "00bf2623da1177b3948143a4ade2f1cda7cb389dee31960861913fa42ef1b00f" + }, + { + "path": "skills/document-skills/pptx/ooxml/scripts/validation/base.py", + "sha256": "f2c70d481613456e32b43869d1604b05c236c8da34b5b3967677a661cac7ba63" + }, + { + "path": "skills/document-skills/pptx/scripts/html2pptx.js", + "sha256": "c675d09a54d6a002e8ca5917b9d24a6568aa8d455bb7abeb212d4f564dd07a34" + }, + { + "path": "skills/document-skills/pptx/scripts/thumbnail.py", + "sha256": "c21fd950b6ada7bd2f029885d3e56bc66b7ff061cc8404c492eb301664aa9e5d" + }, + { + "path": "skills/document-skills/pptx/scripts/rearrange.py", + "sha256": "c04ac37916f398ba621b2d9e1e4c1a69225eaad6d7fb0ad116c237ddeb1b2b68" + }, + { + "path": "skills/document-skills/pptx/scripts/inventory.py", + "sha256": "adead8fe6270e520c397cec9fbee4d606ab10bb80f749e018b42ec894c60d2e5" + }, + { + "path": "skills/document-skills/pptx/scripts/replace.py", + "sha256": "8a590747551be847a904e3296fb2f35aa4e7feeb4970a61596c2375306462820" + }, + { + "path": "skills/document-skills/docx/ooxml.md", + "sha256": "a16f922797eeaa3670ea31c1e49d15b799613d03f39445c857a5dd3221aa3597" + }, + { + "path": "skills/document-skills/docx/docx-js.md", + "sha256": "83b4a2f88d058a10509fbc0b3b12b6933c407805f4d4afc955cd3fb939c16428" + }, + { + "path": "skills/document-skills/docx/SKILL.md", + "sha256": "0bd90681fcab2e282025ee14acf508b60bbd6c41ac6c3bf83c0dc14d52c37933" + }, + { + "path": "skills/document-skills/docx/LICENSE.txt", + "sha256": "79f6d8f5b427252fa3b1c11ecdbdb6bf610b944f7530b4de78f770f38741cfaa" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd", + "sha256": "842e7163409c8d74f4d7088a8bc99500d80bc75332681a0980055b08f374a604" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-2012.xsd", + "sha256": "0fa75578a000439a7988ba0c59fdc69f774bbd416cbacc14d07125b3f686cb74" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-2010.xsd", + "sha256": "568b26ee156cb9549aa439ca2158965f77b7c1602b7e0316f40ac6cf586e35f2" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-cid-2016.xsd", + "sha256": "127ca209fa73d7cb708449cb355c871867948a96e4a74f7bf5811ef62d17991d" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-symex-2015.xsd", + "sha256": "16f6f8072249f431370723c2cd8974672e0d9c897e00e97dd918079df934871b" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-cex-2018.xsd", + "sha256": "fddc2b880cabb9005aebbc7e783e53c19fec1c03df7d0e2f2076a33a0fdfd081" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/microsoft/wml-2018.xsd", + "sha256": "be0ff793a22dd31384650c3a4da14c2fa8062751c2e97b0e5ee852bda13c60ad" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/mce/mc.xsd", + "sha256": "3a37e461ecf5a8670fdec34029703401f8728ab9c96ec1739a6ae58d55212413" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd", + "sha256": "451958454e8588dfc7cd945981ada142ca06ff3307937f5700df059c2b307fa8" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd", + "sha256": "f565adfef5a502044abc3a9153e157edc25af78304d335994afb958874b15e26" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd", + "sha256": "9e0b7209fc69ab11987900404540969976000c5ebe4d4f58c43dc3842886bf3a" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd", + "sha256": "6de111e11403f7cd49027400755bae0ea1cabef2815f09bd40a24f0017613b24" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd", + "sha256": "133c9f64a5c5d573b78d0a474122b22506d8eadb5e063f67cdbbb8fa2f161d0e" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd", + "sha256": "585bedc1313b40888dcc544cb74cd939a105ee674f3b1d3aa1cc6d34f70ff155" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd", + "sha256": "0d103b99a4a8652f8871552a69d42d2a3760ac6a5e3ef02d979c4273257ff6a4" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd", + "sha256": "d173c3e5d61e42e2e3a97226c632fd2ab7cc481fc4e492365b87024ab546daff" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd", + "sha256": "5cb76dabd8b97d1e9308a1700b90c20139be4d50792d21a7f09789f5cccd6026" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd", + "sha256": "41b93bd8857cc68b1e43be2806a872d736a9bdd6566900062d8fdb57d7bbb354" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd", + "sha256": "3fd0586f2637b98bb9886f0e0b67d89e1cc987c2d158cc7deb5f5b9890ced412" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd", + "sha256": "beffeed56945c22a77440122c8bdc426f3fcbe7f3b12ea0976c770d1f8d54578" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd", + "sha256": "6bdeb169c3717eb01108853bd9fc5a3750fb1fa5b82abbdd854d49855a40f519" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd", + "sha256": "c2dd9f61f892deae6acd8d20771ea79b12018af25f3bf8d06639c8542d218cfd" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd", + "sha256": "5d389d42befbebd91945d620242347caecd3367f9a3a7cf8d97949507ae1f53c" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd", + "sha256": "29b254ee0d10414a8504b5a08149c7baec35a60d5ff607d6b3f492aa36815f40" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd", + "sha256": "5375417f0f5394b8dd1a7035b9679151f19a6b65df309dec10cfb4a420cb00e9" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd", + "sha256": "9c085407751b9061c1f996f6c39ce58451be22a8d334f09175f0e89e42736285" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd", + "sha256": "b4532b6d258832953fbb3ee4c711f4fe25d3faf46a10644b2505f17010d01e88" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd", + "sha256": "e2abacbb9a55ce1365f8961bc1b1395bbc811e512b111000d8c333f98458dece" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd", + "sha256": "bdad416b096b61d37b71603b2c949484f9070c830bdaeba93bf35e15c8900614" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd", + "sha256": "475dcae1e7d1ea46232db6f8481040c15e53a52a3c256831d3df204212b0e831" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd", + "sha256": "0b364451dc36a48dd6dae0f3b6ada05fd9b71e5208211f8ee5537d7e51a587e2" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd", + "sha256": "bc92e36ccd233722d4c5869bec71ddc7b12e2df56059942cce5a39065cc9c368" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd", + "sha256": "f5ee623b08b6a66935e5aced2f5d8ad0fc71bf9e8e833cd490150c0fa94b8763" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd", + "sha256": "a539aa2fb154fa50e0f5cc97e6ad7cbc66f8ec3e3746f61ec6a8b0d5d15ecdf2" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd", + "sha256": "12264f3c03d738311cd9237d212f1c07479e70f0cbe1ae725d29b36539aef637" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd", + "sha256": "0ef4bb354ff44b923564c4ddbdda5987919d220225129ec94614a618ceafc281" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd", + "sha256": "7b5b7413e2c895b1e148e82e292a117d53c7ec65b0696c992edca57b61b4a74b" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd", + "sha256": "3213ef1631606250f5010b42cad7ef716f7c59426367798e33c374c0ec391d3a" + }, + { + "path": "skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd", + "sha256": "3c6709101c6aaa82888df5d8795c33f9e857196790eb320d9194e64be2b6bdd8" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/pack.py", + "sha256": "6fe762f45aff8c63fd95b9fcb1337b28921d6fa454e18a0e8158d4c8708d6d00" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/validate.py", + "sha256": "1ec252de8b14b07d16966c48906ccb1c45c68bcd23557ad31d8c50a27f5f8c0f" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/unpack.py", + "sha256": "0bd17f76a1a4c388aba42c6d1d39015fa84e405c3e0692397fe12762bd632b58" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/validation/docx.py", + "sha256": "e65d6cda0525866a24cc847b2e883bd2416ae6f87b3f5b9e2784dfbb0ec13093" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/validation/__init__.py", + "sha256": "83e0f035c5abea238d3f2c3968afbd511ed022b527b7c9cb60a9434cc34ff987" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/validation/redlining.py", + "sha256": "97abfdff4f08f43f9a4bb5c8a2f8fd483398b5b339592724e8635153b5507967" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/validation/pptx.py", + "sha256": "00bf2623da1177b3948143a4ade2f1cda7cb389dee31960861913fa42ef1b00f" + }, + { + "path": "skills/document-skills/docx/ooxml/scripts/validation/base.py", + "sha256": "f2c70d481613456e32b43869d1604b05c236c8da34b5b3967677a661cac7ba63" + }, + { + "path": "skills/document-skills/docx/scripts/__init__.py", + "sha256": "83e262a425814b72add701272b99ddcf9635251c5d4672bf9fc38d2b03f00d85" + }, + { + "path": "skills/document-skills/docx/scripts/document.py", + "sha256": "65f8569034a5893bd5ef0654be5168774fe81c0407b0c4ec80992db9fff91c0c" + }, + { + "path": "skills/document-skills/docx/scripts/utilities.py", + "sha256": "62a4b689056501b91e2df2d1f4e6335818e421c7390e48050717ea8f461a0ed0" + }, + { + "path": "skills/document-skills/docx/scripts/templates/comments.xml", + "sha256": "87e218a3a295016ec855f2cd74495c416072f29c4846e86b527aec0a4d93ba21" + }, + { + "path": "skills/document-skills/docx/scripts/templates/commentsExtensible.xml", + "sha256": "af5d057e16462ca172cea845e502bafb4f3e1b474a8d5848ffe92214853a4935" + }, + { + "path": "skills/document-skills/docx/scripts/templates/commentsExtended.xml", + "sha256": "86bf401354c111102033ed147763faccb82479598f17777a3384c2f3e9fa0014" + }, + { + "path": "skills/document-skills/docx/scripts/templates/commentsIds.xml", + "sha256": "20168f7b237af091332f8348c548eb7f755f583185bb198359c5978155099d67" + }, + { + "path": "skills/document-skills/docx/scripts/templates/people.xml", + "sha256": "61db9900b579acd4c4f84ff7f40df47e77e9e780c40d5f5ef6a7beba41d62ec5" + }, + { + "path": "skills/claude-code/llms.txt", + "sha256": "50010ba6e9dc1ffe6a20a50e55f12c1b0cc00204f756770a9b9ac6f961f7b92b" + }, + { + "path": "skills/claude-code/SKILL.md", + "sha256": "196133ca0328dd48ed5bf62fdeea0f1a37b41dcdece1b3b83fa0baf09fb51905" + }, + { + "path": "skills/claude-code/skill.json", + "sha256": "60cedfbc6c8deaca4816199da27709aaedcf1418b2404d8c6878444f7c8d53b9" + }, + { + "path": "skills/claude-code/references/slash-commands.md", + "sha256": "e7542139e4a2629389f6ce142290224017326f176f04c6c6b90cb8f26cc24262" + }, + { + "path": "skills/claude-code/references/ide-integration.md", + "sha256": "1e599fd5abe9322d22f14e7e178c7aca42292a1fc493e67eb634fcc7c712c56f" + }, + { + "path": "skills/claude-code/references/best-practices.md", + "sha256": "efabe665fd9ba46270e937503648a0f0f0cf6302142ca8ab5e128d66031adf56" + }, + { + "path": "skills/claude-code/references/troubleshooting.md", + "sha256": "39c73df67cce1001f0868671ef99fe19e3607dc40bd1fb6aadc360a66ad0c9a0" + }, + { + "path": "skills/claude-code/references/enterprise-features.md", + "sha256": "8f885147892a5493676f1b97101cd937c3c5800c5a30498d0f0ad8d4a55742e0" + }, + { + "path": "skills/claude-code/references/advanced-features.md", + "sha256": "2fb92f67efbed1338ea53c7be52ed52bffe98f3e1a51c2b79094cc213ca65c2a" + }, + { + "path": "skills/claude-code/references/getting-started.md", + "sha256": "14e1b685b36914f114bfcaa1a74fc618f6aa315d79cc45b76e3234935322bcd1" + }, + { + "path": "skills/claude-code/references/hooks-and-plugins.md", + "sha256": "22231fac9bf3e32fe14a0dcafefb0cbbd7519adcfd350a0d49d0beb2f3736b0a" + }, + { + "path": "skills/claude-code/references/configuration.md", + "sha256": "36f648bd73ee38cfdda861bf4e9366d2110e9ef9a231fdc4084469a88d6b9515" + }, + { + "path": "skills/claude-code/references/cicd-integration.md", + "sha256": "8f74dd8c4f5a197338c5a09fc932fe634eb6a774f1bf2253b872c02f1165c323" + }, + { + "path": "skills/claude-code/references/mcp-integration.md", + "sha256": "29e711f34c06c014f8ea790388b5f089f4c8bb96d73807effdb7b3763d2f6176" + }, + { + "path": "skills/claude-code/references/api-reference.md", + "sha256": "e6c62e7d435c012be4437cd3630b1501bab07ed2183ee5b267dd0f0632f2dfdf" + }, + { + "path": "skills/claude-code/references/agent-skills.md", + "sha256": "b392d1b1eb52915978e7098ef8c8ceb66f510f69b96676201eda325e0b085691" + }, + { + "path": "skills/databases/SKILL.md", + "sha256": "0567d1899bd7fa7530f9a4ac78f38ae597c3ca5adc27958efda4d0052bc03304" + }, + { + "path": "skills/databases/references/postgresql-queries.md", + "sha256": "860725feefff992bdef6defb4acb93235f9662abfd076dafb61c194f4e4ee689" + }, + { + "path": "skills/databases/references/mongodb-indexing.md", + "sha256": "828af24fa18b508b0df3699a0d78345a21a2a6dbb42a6b0522758cc517efce45" + }, + { + "path": "skills/databases/references/postgresql-psql-cli.md", + "sha256": "ca3a81fde3d1e894510b5b906c9e91db2d300df4cbdb1171ad5683df7ccc0b61" + }, + { + "path": "skills/databases/references/mongodb-crud.md", + "sha256": "4c931cea2dbab883fe0c904dc3278c779488ad31cba93dd24b937bd5a2674721" + }, + { + "path": "skills/databases/references/postgresql-performance.md", + "sha256": "77f84195e3f519c8f63679d57d80b1517b04b0188a966141a05cba1d72092116" + }, + { + "path": "skills/databases/references/mongodb-atlas.md", + "sha256": "d37fea35012dc79ba21f9f5f541307b9b7289ec828007b58e02621e63010411b" + }, + { + "path": "skills/databases/references/postgresql-administration.md", + "sha256": "3bf09a45484b4a3633d4b8c0e3812051508116233fcef7136ab21aa696ee479b" + }, + { + "path": "skills/databases/references/mongodb-aggregation.md", + "sha256": "b7cca207376f2a9b62de73615ec8ff22dc339f1cb84fb39b6d548a8271fc6b52" + }, + { + "path": "skills/databases/scripts/db_performance_check.py", + "sha256": "53ca71738a1ac0faa0def8473e75e697ffbe927910b3176ca14dc2f53e819d8b" + }, + { + "path": "skills/databases/scripts/db_migrate.py", + "sha256": "35b08b8a47c423439d34b4f9dced7dd4523d65893f4315d5d3772a99060b2dc3" + }, + { + "path": "skills/databases/scripts/requirements.txt", + "sha256": "da0d7d811e02966201fac258844820dc214cb1b3bcea3006bd94029f6a2518d1" + }, + { + "path": "skills/databases/scripts/.coverage", + "sha256": "e58277507e55c72bd390690e2307f6751cdb9c96e6c6a3ee346f1a710e9c02bd" + }, + { + "path": "skills/databases/scripts/db_backup.py", + "sha256": "ad77002838cfa1e1493ef19208962018251bcc91b6adde571a744e67bc750f2b" + }, + { + "path": "skills/databases/scripts/tests/requirements.txt", + "sha256": "52a7b73d3f2b08178eb3bac41cdc1786f4b06336f741c817d89347e8dbb5504d" + }, + { + "path": "skills/databases/scripts/tests/test_db_backup.py", + "sha256": "ba16ff85557dc29a7a424cb80aebcde760f3a3d59ad7eb4d05a7dfed483d67d7" + }, + { + "path": "skills/databases/scripts/tests/coverage-db.json", + "sha256": "f40c12d1ff34d9c927b31ba3b3f57371f06f45cf462f1516d0eb3f1fa2351b0a" + }, + { + "path": "skills/databases/scripts/tests/test_db_performance_check.py", + "sha256": "f89b7421305587cd45daf1f47d02a0f6c791a92b5a1b5ac0332982f91b6ed2e9" + }, + { + "path": "skills/databases/scripts/tests/test_db_migrate.py", + "sha256": "22f9323d767bdd02a97b7250f14c2cda788f829d9af5497da2417ace21ae2fcc" + }, + { + "path": "skills/template-skill/SKILL.md", + "sha256": "eb685d91de039ed864fbd790cddf31684b017fd4a34ee1a55760d8d7cdbadefa" + }, + { + "path": "skills/devops/SKILL.md", + "sha256": "c947d0a2f13d527a4e54edd782b1fc6f726288a4a45002b1e0a64d0d7efe878e" + }, + { + "path": "skills/devops/.env.example", + "sha256": "2e90b03a2ca418a05985b6cc4c2a02ccb96764b7b058538597217f1311c7bf18" + }, + { + "path": "skills/devops/references/cloudflare-workers-advanced.md", + "sha256": "fc4b2c5bd5c826ce1dbbeec914666dd6c8edc64287dbd0acc75c4de1ccaae019" + }, + { + "path": "skills/devops/references/cloudflare-r2-storage.md", + "sha256": "03c90dd24354676d97ac53b2b111daae3c88002955849f2bddd9689447db4235" + }, + { + "path": "skills/devops/references/cloudflare-d1-kv.md", + "sha256": "d35c07d3bf5356a21544e088c9995cd59dec463992ae95e7a848130ebf8c6d3d" + }, + { + "path": "skills/devops/references/cloudflare-platform.md", + "sha256": "d009a10f0f37073a08ae51ab6768a63b3baf20f815c2048e3370dfd263d21e6a" + }, + { + "path": "skills/devops/references/browser-rendering.md", + "sha256": "d258f97511e47ef425bfc1ae5578720162fa2499466ef4c275072c1a4db25d9d" + }, + { + "path": "skills/devops/references/gcloud-services.md", + "sha256": "2581140e4bf5412fdc7b2758e9c2acc59e428117af26b1cc1efb10a21106c454" + }, + { + "path": "skills/devops/references/cloudflare-workers-apis.md", + "sha256": "d36537cf342f93b3db0bc6457f2e61ac5f10e3daf82fab2d557cdb88278fe57f" + }, + { + "path": "skills/devops/references/cloudflare-workers-basics.md", + "sha256": "a975a4f0ef7e663e66f54156153568b2735a9a626e0fae13cd777e5399979a37" + }, + { + "path": "skills/devops/references/docker-compose.md", + "sha256": "cf6eb6338aa55201f0a5b4390cdb33d4de2b8cffabf2c2b84dac612a2d4f0816" + }, + { + "path": "skills/devops/references/docker-basics.md", + "sha256": "9e92f9e2f9e147d1002b3abdb45526ac175fd63d1fec6b5862a3c3c0cb828631" + }, + { + "path": "skills/devops/references/gcloud-platform.md", + "sha256": "195c09f69bb47976467fe1e5094a156f16ac88ad8e20003b60d7f9ae037bc09d" + }, + { + "path": "skills/devops/scripts/requirements.txt", + "sha256": "4a508a3b086c25ad5adc9b743a8c0a576910e650008b8144fc58f68770b662fd" + }, + { + "path": "skills/devops/scripts/cloudflare_deploy.py", + "sha256": "439844726c149c7dd5c7a6fb85b20059b8e230041c1a61364fd609b4c503216c" + }, + { + "path": "skills/devops/scripts/docker_optimize.py", + "sha256": "7e484e26109df1790ef7c31269647aa62e0be519df6fbfe558a0df212ebe12fd" + }, + { + "path": "skills/devops/scripts/tests/requirements.txt", + "sha256": "0795bdcfb80afae0ff06e9cbe5bed67b39283e577984bed87d85da3fec798a37" + }, + { + "path": "skills/devops/scripts/tests/test_docker_optimize.py", + "sha256": "c82fd9d4f18e60d1ebdbcebda5781d462a6a0c846658512aa5239e1eaee0d9fb" + }, + { + "path": "skills/devops/scripts/tests/test_cloudflare_deploy.py", + "sha256": "66373336b3290bed9ca300699816954006cabc9f91897cb4759358d365dabc69" + }, + { + "path": "skills/ui-styling/SKILL.md", + "sha256": "7f5b4a0ce2a109779d6ca235543ee7755881c53f0258d526e174bd37fd4c8dd8" + }, + { + "path": "skills/ui-styling/LICENSE.txt", + "sha256": "58d1e17ffe5109a7ae296caafcadfdbe6a7d176f0bc4ab01e12a689b0499d8bd" + }, + { + "path": "skills/ui-styling/canvas-fonts/Lora-Italic.ttf", + "sha256": "be627e595184e8afe521f08da0607eee613f1997d423bc8dadc5798995581377" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSerif-Italic.ttf", + "sha256": "9c86e4d5a47b50224a2463a9eca8535835257c8e85c470c2c6b454b1af6f046e" + }, + { + "path": "skills/ui-styling/canvas-fonts/WorkSans-Regular.ttf", + "sha256": "e67985a843df0d3cdee51a3d0f329eb1774a344ad9ff0c9ab923751f1577e2a4" + }, + { + "path": "skills/ui-styling/canvas-fonts/ArsenalSC-OFL.txt", + "sha256": "8ddd61b18ba2c0d0dbe4a691cf5f1a0673f473d02fa0546e67ee88c006aeff6e" + }, + { + "path": "skills/ui-styling/canvas-fonts/Italiana-OFL.txt", + "sha256": "8373b11312ace78c4cec2e8f9f6aa9f2330601107dac7bcf899c6f2dbd40c5a5" + }, + { + "path": "skills/ui-styling/canvas-fonts/CrimsonPro-Italic.ttf", + "sha256": "52318db3526b644e6efa60be0b3ca5a50e40fbe8bd026c261e0aa206f0772267" + }, + { + "path": "skills/ui-styling/canvas-fonts/Outfit-OFL.txt", + "sha256": "1945b62cd76da9a3051a1660dde72afaa64ffc2666d30e7a78356d651653ba2f" + }, + { + "path": "skills/ui-styling/canvas-fonts/Gloock-OFL.txt", + "sha256": "c0a3f3125ac491ef3d1f09f401be4834c646562f647e44f2bcbc49f0466c656d" + }, + { + "path": "skills/ui-styling/canvas-fonts/RedHatMono-Regular.ttf", + "sha256": "452fe826871b37539f5212b20c87cf30f82f58dd2741f1c96edd1dcbdc0db6b4" + }, + { + "path": "skills/ui-styling/canvas-fonts/YoungSerif-OFL.txt", + "sha256": "cdcb8039606b40a027a6d24586ec62d5fe29c701343d82a048c829cb28a3dd28" + }, + { + "path": "skills/ui-styling/canvas-fonts/Outfit-Bold.ttf", + "sha256": "6654b93d21301ec61887d3cedd6c11d9df1b1dfb63f9cf45ac7995f6e2235ab1" + }, + { + "path": "skills/ui-styling/canvas-fonts/Boldonse-Regular.ttf", + "sha256": "cc2e540604565c0f90a7d8d46194a2f42fc9c45512cd2e39bf03b50eb68c35a4" + }, + { + "path": "skills/ui-styling/canvas-fonts/CrimsonPro-OFL.txt", + "sha256": "35680d14547b6748b6f362a052a46d22764ce5eccf96e18b74f567bb2ee58114" + }, + { + "path": "skills/ui-styling/canvas-fonts/RedHatMono-Bold.ttf", + "sha256": "7ef48353f4be5ddb90f000f6fad48f2b62b3e8c27d9818d8d45ff46c201065e0" + }, + { + "path": "skills/ui-styling/canvas-fonts/PixelifySans-OFL.txt", + "sha256": "7f54d1d9f1ae1ba9f2722f978145f90324fea34ca3c2304b3a29cfa96ac6037e" + }, + { + "path": "skills/ui-styling/canvas-fonts/Jura-OFL.txt", + "sha256": "eaf9bdb675f6d87e5feb88199ab3ea581d3bd2082f426e384fa9c394576d7260" + }, + { + "path": "skills/ui-styling/canvas-fonts/Outfit-Regular.ttf", + "sha256": "f24945365147c9e783e91d8649959b59be6b00c9ee4ecd2f6b33afbb2dd871fe" + }, + { + "path": "skills/ui-styling/canvas-fonts/Silkscreen-OFL.txt", + "sha256": "6b849745119bbe85ec01fd080c9cd50234da9f52ac6e48b55d1a424a0c4d7ca9" + }, + { + "path": "skills/ui-styling/canvas-fonts/JetBrainsMono-Bold.ttf", + "sha256": "a2349098b9e45419e7bf0e2958d6c4937a049dded37387b08be725be4c7615f3" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexSerif-Regular.ttf", + "sha256": "77cd233a2af8dc6b1022faea3bb3b01f3c75af68bcf530cb6aeb15982ff3dbb7" + }, + { + "path": "skills/ui-styling/canvas-fonts/YoungSerif-Regular.ttf", + "sha256": "f8dc08f77abad753a00670af70756a8ace938e5c3f0b770f4f4c2071c4bd8fc6" + }, + { + "path": "skills/ui-styling/canvas-fonts/Italiana-Regular.ttf", + "sha256": "15c4dd6ab8cf4a29ba8826f65edcbe2f6c266c557d34d081f25072dfd5605fd2" + }, + { + "path": "skills/ui-styling/canvas-fonts/Lora-Bold.ttf", + "sha256": "7d74015e950c2fb66519c7295b8155621d22200ae2ca2a4c6b43ce3c490cac87" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexSerif-BoldItalic.ttf", + "sha256": "da64b75f4284f53e7b5c71fa190a35b8bf3494fe19f1804c81c3a53340bca570" + }, + { + "path": "skills/ui-styling/canvas-fonts/JetBrainsMono-Regular.ttf", + "sha256": "b6b1ff4ddefe36d7f2a6174e1d001cab374e594519ee9049af028d577b64c5f5" + }, + { + "path": "skills/ui-styling/canvas-fonts/CrimsonPro-Regular.ttf", + "sha256": "48fad08cb1917a7b2f2c6fe5135d6c07743a6663cf7631ec4481108aaf081422" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSans-Italic.ttf", + "sha256": "78e85858e371b2cb4e18f617c10f0f937c0e12a0887ffee98555b24ed305b3a7" + }, + { + "path": "skills/ui-styling/canvas-fonts/Jura-Light.ttf", + "sha256": "c891a381df056b2c4dfe85841e911bf45da0890fa21a7b2692cbe5ea1f505e1e" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexMono-Bold.ttf", + "sha256": "dbd2a2fb024579438d6400a84e57579bfd2dbe67c306c8fd9fde92a61e4f2eea" + }, + { + "path": "skills/ui-styling/canvas-fonts/WorkSans-BoldItalic.ttf", + "sha256": "a5b2cad813df0aaa7d16621f2e93b5117c25e9bc788bc9a3ad218e9d6348ce34" + }, + { + "path": "skills/ui-styling/canvas-fonts/WorkSans-OFL.txt", + "sha256": "ace8c22a3326318b54e67c3691857929634205533f454a70ef5a3473ddb2e2ba" + }, + { + "path": "skills/ui-styling/canvas-fonts/Lora-OFL.txt", + "sha256": "62e37a82d3f1ef2a70712885fa8b3144b65fd144d8e748d6196b690a354d792c" + }, + { + "path": "skills/ui-styling/canvas-fonts/Silkscreen-Regular.ttf", + "sha256": "49567408600809e25147e9225ac4f37f410e2df45a750696c45027531fb65f1b" + }, + { + "path": "skills/ui-styling/canvas-fonts/PixelifySans-Medium.ttf", + "sha256": "38397504f71c122b03d234ea6f55118e3d5bdbddffd82bedddbd7755d3b3be82" + }, + { + "path": "skills/ui-styling/canvas-fonts/PoiretOne-Regular.ttf", + "sha256": "9cf265b139648b36b6c0afdfeb0bf27f7e66db9a16094bc40f644d8da05bc318" + }, + { + "path": "skills/ui-styling/canvas-fonts/BricolageGrotesque-Regular.ttf", + "sha256": "972a6d098c9867ae131d0ea99e221e63976b11a19d4b931c2c7ace525674e4f6" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexMono-Regular.ttf", + "sha256": "ab08018ccd276b79fb2c636bb95b9c543598f9d50505fe92506fcb4dae7810cd" + }, + { + "path": "skills/ui-styling/canvas-fonts/CrimsonPro-Bold.ttf", + "sha256": "48f191e38355c8db100eb3ce157c20f9302a3b9a37b44a660f77ecfce3986609" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSerif-Regular.ttf", + "sha256": "56ac3be03ac3ba283196b3e77850ab2ffcf56cfb6fd3212c5620109a972f8c99" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSans-OFL.txt", + "sha256": "bf4dc6d13a8cccd4807133c77a1ee9619a16b92cb23322258725ab6731c2f6e5" + }, + { + "path": "skills/ui-styling/canvas-fonts/Tektur-OFL.txt", + "sha256": "3f1466cb5438f31782eeb6e895f3a655bc4d090e24263e331f555357d1cb734e" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSans-BoldItalic.ttf", + "sha256": "3762f6cef95d6039489ad5ba5787d4c30f17a1ad01e9ac3c816ed69692722a68" + }, + { + "path": "skills/ui-styling/canvas-fonts/EricaOne-OFL.txt", + "sha256": "e0de629968b52255548d5fafcf30b24ff9edae0eda362380755a75816404d0fa" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexSerif-Italic.ttf", + "sha256": "b11f1048745e715a55c9d837b3f10226ca3d78867b7db7251ddad8f98dcf0f38" + }, + { + "path": "skills/ui-styling/canvas-fonts/DMMono-Regular.ttf", + "sha256": "f98ada968dc3b6b2c08d3f5caaf266977df0bfe0929372b93df5a06cf2ace450" + }, + { + "path": "skills/ui-styling/canvas-fonts/NationalPark-Bold.ttf", + "sha256": "69ac4c301c4a7233c6e602d12a92c54d7967b575f4449951c45ce773f7acff53" + }, + { + "path": "skills/ui-styling/canvas-fonts/DMMono-OFL.txt", + "sha256": "bfe7842fcb88323e2981e24710c25202677385a8c75fb6a87217b275a0247ae3" + }, + { + "path": "skills/ui-styling/canvas-fonts/NationalPark-OFL.txt", + "sha256": "81c6c71d83b5b45d7344f96df12bb4a2477a5b092a9144757ee1d0f50f855175" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSans-Regular.ttf", + "sha256": "a22cb26e48fd79bcb01bf2fc92d36785474dce36d9c544ab0a8868c2657c4a87" + }, + { + "path": "skills/ui-styling/canvas-fonts/LibreBaskerville-Regular.ttf", + "sha256": "2101302538d9e88adb679031c04623e4578b5745e89566284fd2c508d79acae0" + }, + { + "path": "skills/ui-styling/canvas-fonts/EricaOne-Regular.ttf", + "sha256": "db1d89e80e33a8a01beaaac7a85df582857d24a43f1e181461aa7ff5d701476a" + }, + { + "path": "skills/ui-styling/canvas-fonts/SmoochSans-OFL.txt", + "sha256": "74c9c4eb88e891483e1b7bc54780b452cbf4f4df66d4e71881d7569aa2130749" + }, + { + "path": "skills/ui-styling/canvas-fonts/NothingYouCouldDo-Regular.ttf", + "sha256": "d866f985896d3280f4fce72db7e17302c24a0c1fdb0699b6b5ed3af14f944d57" + }, + { + "path": "skills/ui-styling/canvas-fonts/WorkSans-Italic.ttf", + "sha256": "6b7f7002e0b0c8b261fe878658ef5551e3e59d9f6b609b04efb90dde1e2c1ada" + }, + { + "path": "skills/ui-styling/canvas-fonts/LibreBaskerville-OFL.txt", + "sha256": "55959eef5b0c3b2e3c1c7631b8ff0f9447d75de20f29cfa7db5bcfb026763343" + }, + { + "path": "skills/ui-styling/canvas-fonts/PoiretOne-OFL.txt", + "sha256": "2eaf541f7eb8b512e4c757a5212060abf5b6edfef230e9d7640bf736b315c33a" + }, + { + "path": "skills/ui-styling/canvas-fonts/Gloock-Regular.ttf", + "sha256": "e86b4ce66dbd3f1f83eee8db99ec96e0da1128c3f53df0e9b3b7472025dfe960" + }, + { + "path": "skills/ui-styling/canvas-fonts/BricolageGrotesque-OFL.txt", + "sha256": "0e4f4eb8534bc66a76aca13dd19c1f9731b2008866b29ccff182b764649df9b4" + }, + { + "path": "skills/ui-styling/canvas-fonts/SmoochSans-Medium.ttf", + "sha256": "dd76e6e77cce82f827a8654cd906e9ce58f3aaf78adda63c4a7f655b8ecb41f0" + }, + { + "path": "skills/ui-styling/canvas-fonts/NothingYouCouldDo-OFL.txt", + "sha256": "7c2a6970584ddad04919816163746f83b378078015899b18468b40f05e9ce128" + }, + { + "path": "skills/ui-styling/canvas-fonts/Tektur-Regular.ttf", + "sha256": "162e1b36c4718c5b051b36c971ad7e50d341944f35618f480422ebbe72988f98" + }, + { + "path": "skills/ui-styling/canvas-fonts/GeistMono-Regular.ttf", + "sha256": "a55c1b51cda4afeab9e471e7947b85a20f7c8831d7e6b1470c1b7fbdc0f0f15e" + }, + { + "path": "skills/ui-styling/canvas-fonts/Lora-BoldItalic.ttf", + "sha256": "152f87e71f5ddb60d5c57ecd9132807c947e65c42977193c9164e7c5a6690081" + }, + { + "path": "skills/ui-styling/canvas-fonts/RedHatMono-OFL.txt", + "sha256": "435fbfb7e66988b2a06686a4cb966faec733f35d8fe100a1601573c27f3e0bb8" + }, + { + "path": "skills/ui-styling/canvas-fonts/BricolageGrotesque-Bold.ttf", + "sha256": "a737b146fe0d77ffe8a86e3cd16700dd431d3b1e420d4fd80e142cd68a1cb50d" + }, + { + "path": "skills/ui-styling/canvas-fonts/NationalPark-Regular.ttf", + "sha256": "a477338b7e18308d476650dfe31235ef86a883572665e56ffb5fb80f82009b58" + }, + { + "path": "skills/ui-styling/canvas-fonts/BigShoulders-Regular.ttf", + "sha256": "18a879fc71978a4447150705caf880a9da3860083c259fd29e6dc03057b6842a" + }, + { + "path": "skills/ui-styling/canvas-fonts/GeistMono-Bold.ttf", + "sha256": "75c0828d5c1ee44b9ef9f4df577bf41595ec362e2ea3f1e558590c9e92c7949d" + }, + { + "path": "skills/ui-styling/canvas-fonts/GeistMono-OFL.txt", + "sha256": "6a873c900f584109b13ae0aaf81d6e3cf0a68751a216b03f7b6c68d547057bb4" + }, + { + "path": "skills/ui-styling/canvas-fonts/Lora-Regular.ttf", + "sha256": "7ed00e7c9cdf16ab7e2fd2361fe45d4f0b61263cd60aae398b27b7ee08108827" + }, + { + "path": "skills/ui-styling/canvas-fonts/ArsenalSC-Regular.ttf", + "sha256": "65e6f89df58f68fd905b3add34a79dd6106aa3b3044df0dad9676fff53d504b9" + }, + { + "path": "skills/ui-styling/canvas-fonts/JetBrainsMono-OFL.txt", + "sha256": "a76abf002c49097d146e86740a3105a5d00450b1592e820a1109a8c5680cd697" + }, + { + "path": "skills/ui-styling/canvas-fonts/BigShoulders-OFL.txt", + "sha256": "fbc746aabf0eb1847dfd92e2efc4596d79fa897d60b8e64062a22f585508fb3f" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexMono-OFL.txt", + "sha256": "5294ce778857e1eb02e830b6ab06435537d38f43055327e73d03a2d4d57d5123" + }, + { + "path": "skills/ui-styling/canvas-fonts/Tektur-Medium.ttf", + "sha256": "52bbe8c9b057b3d2da4eeace31a524b1ea26a1375ae34319cf6900ccc57a4c82" + }, + { + "path": "skills/ui-styling/canvas-fonts/Jura-Medium.ttf", + "sha256": "c72965cb732a92872643819fd1734128238583cc36b116313859137a51d3368a" + }, + { + "path": "skills/ui-styling/canvas-fonts/InstrumentSans-Bold.ttf", + "sha256": "444f85bf1c4b0e1ce1ca624f6be54bcd832207714ccaf4ea99ee531341683bdf" + }, + { + "path": "skills/ui-styling/canvas-fonts/Boldonse-OFL.txt", + "sha256": "45cc82ab4032273c0924025ffcf8f0665a68e1a5955e3f7247e5daf1deeb1326" + }, + { + "path": "skills/ui-styling/canvas-fonts/BigShoulders-Bold.ttf", + "sha256": "b43bcd198b9fdf717dd42aa61a34dba32e01aceaeae659d689afd0ca52c37ea2" + }, + { + "path": "skills/ui-styling/canvas-fonts/IBMPlexSerif-Bold.ttf", + "sha256": "b8d294e9b5c5a0940f167c3ced0f7ef2e3f57082ca3ff096ef30e86e26c1c159" + }, + { + "path": "skills/ui-styling/canvas-fonts/WorkSans-Bold.ttf", + "sha256": "240d125fc9f8561363dc1ea3f513501253bd70942f41468f48f0b0cafb0c82e2" + }, + { + "path": "skills/ui-styling/references/shadcn-theming.md", + "sha256": "d17d641474221123ff0e0288043ca2f2dab3ac96908a19eed28d82f9e58865ed" + }, + { + "path": "skills/ui-styling/references/tailwind-utilities.md", + "sha256": "aba1c40ef84f43beea142ae310371e332541f39d973df90a71f1854823effda1" + }, + { + "path": "skills/ui-styling/references/tailwind-responsive.md", + "sha256": "8d00ae620df26daea4623c7bf996d6e612e09d359305fcfa2b40cab757ed840c" + }, + { + "path": "skills/ui-styling/references/tailwind-customization.md", + "sha256": "4c5adeed6263a274f74f6eb2c816ca420fd6ef4f35683582988bacd404799afe" + }, + { + "path": "skills/ui-styling/references/shadcn-accessibility.md", + "sha256": "a22cd4ccf82b635b2b2c4f12416a0c4f7edcd825d219d6089450fa797a0d9511" + }, + { + "path": "skills/ui-styling/references/shadcn-components.md", + "sha256": "79c4f91cbf68993a43fa3a95889c35db73de45bdd3dfd83a6830863c01aa598a" + }, + { + "path": "skills/ui-styling/references/canvas-design-system.md", + "sha256": "f5de85ff39d9f3a8275c4164b921ba72d5e1d34b8d7a9243d9ee34b3870a4f5f" + }, + { + "path": "skills/ui-styling/scripts/tailwind_config_gen.py", + "sha256": "fd0e7b96fdb0d24b9ac85e87aed1dae6a2399b6c508809b3db71a32696584511" + }, + { + "path": "skills/ui-styling/scripts/requirements.txt", + "sha256": "09402d2d274248e18bb5fd0a0267fd7cbf99b7a2440e73e87f054aa8f259f9da" + }, + { + "path": "skills/ui-styling/scripts/.coverage", + "sha256": "a1f68d7a1d6322975c40c893004bc5e811d455d3ab124dab5154aad7060e66f5" + }, + { + "path": "skills/ui-styling/scripts/shadcn_add.py", + "sha256": "0bcdf28ea2c2d5d4f17a2b1e0a0cab29ff649b9db7b134ecc6e9898a3d84824f" + }, + { + "path": "skills/ui-styling/scripts/tests/requirements.txt", + "sha256": "80846c98ee02a7e9651ec4eee6709f840b785646b469aad5284ab7ab9b344f17" + }, + { + "path": "skills/ui-styling/scripts/tests/test_shadcn_add.py", + "sha256": "4012b1efe1e51a5dcf31078b0d9069f92cfe2045e9a35dd52fdd4631b1815150" + }, + { + "path": "skills/ui-styling/scripts/tests/test_tailwind_config_gen.py", + "sha256": "2a8b7b3e8b9ff081532551f2168dd08a8471e71c17747ae8d6026e1273068dc8" + }, + { + "path": "skills/ui-styling/scripts/tests/coverage-ui.json", + "sha256": "33bdc1f5998db9a38cac8e6128a82711f8f8b65fbb3dcb0c1680ae260441cbce" + }, + { + "path": "skills/code-review/SKILL.md", + "sha256": "d7ae241ea6db2cf782b4d4c07374dad7f9dfa8a7882a147668b9f26fe72bac9b" + }, + { + "path": "skills/code-review/references/code-review-reception.md", + "sha256": "2f6114025609bed0dcf13a9b4a583f625c9f27b91052c9c90a8833af9a83c804" + }, + { + "path": "skills/code-review/references/requesting-code-review.md", + "sha256": "a3703cacc5c72bf8e39759c11dca0c1204008a53e9704bca7be5d016e4d14a26" + }, + { + "path": "skills/code-review/references/verification-before-completion.md", + "sha256": "963f6b63d58d88e029181126528a1c720f76c4f8cfaec3532ece14795aef5b77" + }, + { + "path": "skills/debugging/systematic-debugging/test-pressure-1.md", + "sha256": "0b6a915db0054577819834c79be9eb614e97bddba10d73768e1fbe91cfed048a" + }, + { + "path": "skills/debugging/systematic-debugging/test-pressure-2.md", + "sha256": "b2030aeffba07050e8ad573ddf87486457c4a016a786bb326235bebd856f2016" + }, + { + "path": "skills/debugging/systematic-debugging/CREATION-LOG.md", + "sha256": "b482ef9a918fbfc6c369729e8160633ddfa2332466dd362ee73f1527c239ef8b" + }, + { + "path": "skills/debugging/systematic-debugging/test-academic.md", + "sha256": "fe2ba480d78ac0d686dc025f41c2a32a43d642bf533f91b0c6053a04d35d6486" + }, + { + "path": "skills/debugging/systematic-debugging/SKILL.md", + "sha256": "48252adfa183d927befe3894d7aa32c23b59d374cde101f4e2269e14c7c1c5df" + }, + { + "path": "skills/debugging/systematic-debugging/test-pressure-3.md", + "sha256": "96b50a52e2c7989c9cf20fb752c47c1e9a3a70dc362f8f7989f8f5b64dac7708" + }, + { + "path": "skills/debugging/root-cause-tracing/SKILL.md", + "sha256": "476830e10488843688d17d13feae60ca040c769939971e8b8554f67cffb7ddf8" + }, + { + "path": "skills/debugging/root-cause-tracing/find-polluter.sh", + "sha256": "f4dc594206175b17de25464b5f60a0e011774a7c7843014b6442338a085eba57" + }, + { + "path": "skills/debugging/verification-before-completion/SKILL.md", + "sha256": "6c3c414b4058c5d6a461abfcee62aebac50741488e9b327445bef661d1634fda" + }, + { + "path": "skills/debugging/defense-in-depth/SKILL.md", + "sha256": "26b67533bbd6b088d1004f027d2305851270794cb4c632d897abe934956a6a0e" + }, + { + "path": "skills/shopify/README.md", + "sha256": "c13775e03e078aae7fee1956ec758e2f728195495a1033680e4c202625bb2d64" + }, + { + "path": "skills/shopify/SKILL.md", + "sha256": "9a6b1a96e3d84a58d8264560daf5d03f3626fa25dcd4739afc99ccd76cbfd398" + }, + { + "path": "skills/shopify/references/extensions.md", + "sha256": "8264fa2dac7d21493fe23cebe96271fcf90f88119b49491157c29a3fbcdb16e9" + }, + { + "path": "skills/shopify/references/app-development.md", + "sha256": "af92f5533dec1d23a1836a84e253d2f3737f0f00db03146ab9d1ca171cb05e10" + }, + { + "path": "skills/shopify/references/themes.md", + "sha256": "0a18523f8183f060d4867bec17a4f25f48f7e8cdbc0b6e543795dd00fa087c67" + }, + { + "path": "skills/shopify/scripts/requirements.txt", + "sha256": "ff02bd21c424070c78b466debfc2748ea3f42f43c5d4bd888c6be2c38e3bee45" + }, + { + "path": "skills/shopify/scripts/.coverage", + "sha256": "9b1a52fa7e42d18f69c31e808c40fb0fe51a4f60da6a9e4696bed1274cc530fb" + }, + { + "path": "skills/shopify/scripts/shopify_init.py", + "sha256": "ec668b243ff55b78437486ce8e99b6a8bd92a4b7ea63b21ee724c5caba960521" + }, + { + "path": "skills/shopify/scripts/tests/.coverage", + "sha256": "d2b0dfe017df2674b062e577e2b1b1b93651e18353d78e9aede6ed14db6fafd8" + }, + { + "path": "skills/shopify/scripts/tests/test_shopify_init.py", + "sha256": "7bb517b08c58b2b1b9b10f89f1437dba378188adfe63e93aa1dccb3abbeabbd3" + }, + { + "path": "skills/mcp-management/README.md", + "sha256": "171e3485f70e6e21dd9512c3fb264bfc54081cc5db2ba710b16d3e200f427c03" + }, + { + "path": "skills/mcp-management/SKILL.md", + "sha256": "e163d352e168a42a3403d2a71846914e4d2a6648ccd4e5b8fc90a83004c7cf10" + }, + { + "path": "skills/mcp-management/references/mcp-protocol.md", + "sha256": "eb4ab38de303d0e9702d5e5f82953e89ad156755b446d07af6fbfd55503d3579" + }, + { + "path": "skills/mcp-management/references/configuration.md", + "sha256": "82e722baef392dd1e80702226263c20cb38e63ba296132d99779b8bdc5dd36cf" + }, + { + "path": "skills/mcp-management/references/gemini-cli-integration.md", + "sha256": "4e8123a392da44a996b8a2c523b0780d517640926ba793aa0eea5fd4286bc6e2" + }, + { + "path": "skills/mcp-management/scripts/cli.ts", + "sha256": "b23345ecd42a85c0c24bb4367ea37e1d0baaf4b708e2cfdb8986c28ec42bb0d9" + }, + { + "path": "skills/mcp-management/scripts/mcp-client.ts", + "sha256": "6f1ad0f5a0e80cc1150a3197b801bfeff5aca2e61087561b2ba24934eb597e97" + }, + { + "path": "skills/mcp-management/scripts/package.json", + "sha256": "38ccee009ffbd44163256fb5e8ee1702db749adb25a7fe65e2a84ad88ae7f78c" + }, + { + "path": "skills/mcp-management/scripts/tsconfig.json", + "sha256": "c9d43a3e50d31af519710feb01ae2558001abd84bdb87ddc08628c193fe273a6" + }, + { + "path": "skills/mcp-management/scripts/.env.example", + "sha256": "58b814af626ee7d10e24653156c0b5df7c7ee236a2757049e0c7a2e99b60fdd1" + }, + { + "path": "skills/mcp-management/assets/tools.json", + "sha256": "0ce5ca4e12dfa1738cd1a54697b697c9bd5cebf6fb42a8f452a9ea7b1cb8eca2" + }, + { + "path": "skills/problem-solving/ABOUT.md", + "sha256": "5ac236d43b2f1173d0fbc7a80247848b4f6454b89d826addb7e5aa9c804d50d0" + }, + { + "path": "skills/problem-solving/when-stuck/SKILL.md", + "sha256": "3f5dd9d0dee4703d7f65e565a751d6a6336b9a99aeda5af15bbba8e9c6561c4a" + }, + { + "path": "skills/problem-solving/simplification-cascades/SKILL.md", + "sha256": "57417c2d46934c2214b3698823b7b1098c84fc2bcd5dea372ea2ba9fc00f4c32" + }, + { + "path": "skills/problem-solving/collision-zone-thinking/SKILL.md", + "sha256": "416850bf47da72973a0891c5b9433b22c3b5677ec97f07742045c5a1ef00dd6f" + }, + { + "path": "skills/problem-solving/meta-pattern-recognition/SKILL.md", + "sha256": "2a579c471488a1ae73044bf6bf07d4fe8ccf63bcfdb3158e0059f815fd2f6eb5" + }, + { + "path": "skills/problem-solving/inversion-exercise/SKILL.md", + "sha256": "01786f5fd93aabb4a4beb14f88104a2d16d57ffdf5f2a20054e17dfede1c9215" + }, + { + "path": "skills/problem-solving/scale-game/SKILL.md", + "sha256": "43373c54188b3bd9ba5eca74e57a53dfd3967ed65503996bcd16441e34cd15aa" + }, + { + "path": "skills/common/api_key_helper.py", + "sha256": "3d7dbdf19a835ee0467d8bde2cdbd5fcf42ce45f0c4d84a7a844268b4d76ff88" + }, + { + "path": "skills/common/README.md", + "sha256": "c5f3373de60c0bfc530c73a6f2392e06263b7fa425894a139910941de53fd777" + }, + { + "path": "skills/skill-creator/SKILL.md", + "sha256": "a1208de184ff0607e8eade5defc90ebcef6a0c76ab45c4b25b3b4c0c240dd631" + }, + { + "path": "skills/skill-creator/LICENSE.txt", + "sha256": "58d1e17ffe5109a7ae296caafcadfdbe6a7d176f0bc4ab01e12a689b0499d8bd" + }, + { + "path": "skills/skill-creator/scripts/init_skill.py", + "sha256": "0bba250b94caa4cb2b28b15dad26fdcf371aeda4c9797b8120e55c2e33e0c73c" + }, + { + "path": "skills/skill-creator/scripts/package_skill.py", + "sha256": "692525dd8096aee51730ab142fb2605894a6f1a4135bcffebbf4f8ed3cf8715e" + }, + { + "path": "skills/skill-creator/scripts/quick_validate.py", + "sha256": "41647b79d15e7e51b944d56843e9f87a33f4c66e64ded5964d1f95bad2fe634a" + }, + { + "path": "skills/media-processing/SKILL.md", + "sha256": "eb2c53742a9092234ad499be028adcf976bc2d6578e513092361ea3cda40d016" + }, + { + "path": "skills/media-processing/references/format-compatibility.md", + "sha256": "900ac7216f595d94ae597096c2bf12cd62bfe0cf1841b143e4ac8564cd12e02f" + }, + { + "path": "skills/media-processing/references/ffmpeg-encoding.md", + "sha256": "903682c3375524ce29a12f52c72e947ab114cee1187d88652360911d06a5759f" + }, + { + "path": "skills/media-processing/references/imagemagick-batch.md", + "sha256": "e9aefed340939dc2241df5a45f82a49516bd348b6340d2acb42b4a3cfdf30537" + }, + { + "path": "skills/media-processing/references/imagemagick-editing.md", + "sha256": "61f9e9f4d7266db2ed6bd7774f4790c4146fab07c127413a1037b53af8096c0e" + }, + { + "path": "skills/media-processing/references/ffmpeg-streaming.md", + "sha256": "087eab32cf7336e7815158635a4487db9640d038a22bb01a269277393c5b510f" + }, + { + "path": "skills/media-processing/references/ffmpeg-filters.md", + "sha256": "347447ae305410e5bb72a4934ad4fadeb91fdc12435551b038a7f895a49a72a0" + }, + { + "path": "skills/media-processing/scripts/requirements.txt", + "sha256": "656461e5a959cc78eda21807a73b5c20e78e6bd116fa89c7606c82eeacaab221" + }, + { + "path": "skills/media-processing/scripts/video_optimize.py", + "sha256": "c2df1f79fc8314b94bade5e7ceee7b4d54459402a7f91bf12ebe71b640746b68" + }, + { + "path": "skills/media-processing/scripts/media_convert.py", + "sha256": "896961d11de1f7040243e388a82b30295e3224ec7b0afb4d023f4579c832e709" + }, + { + "path": "skills/media-processing/scripts/batch_resize.py", + "sha256": "ce8e55a5546a0c49b84c8e97021a875db7be83f2f8f1a53ee1285e64d4e0b60e" + }, + { + "path": "skills/media-processing/scripts/tests/requirements.txt", + "sha256": "7f336e73b484fac1a0807a6cfba48eefe79c12f3c348d988a708dda2d6df6d14" + }, + { + "path": "skills/media-processing/scripts/tests/.coverage", + "sha256": "86e832012e03a2fbebbed563c84e3fbfae6f6fa379db99696301593fa40790bd" + }, + { + "path": "skills/media-processing/scripts/tests/test_video_optimize.py", + "sha256": "483f996459f251f9f15affaca183e9bee558432a7b49903a87fff9d7da244273" + }, + { + "path": "skills/media-processing/scripts/tests/test_batch_resize.py", + "sha256": "ce898f6a8f10596399a4dfaadb1abc9e2a133e18e375e1b6a5b5e56faf6d0033" + }, + { + "path": "skills/media-processing/scripts/tests/test_media_convert.py", + "sha256": "c067e325304b7cb84701a1481e2aa077ce8a29a04799a1b31a7768342d4c1f20" + }, + { + "path": "skills/chrome-devtools/SKILL.md", + "sha256": "893563ea6ee597b247f289589aeb6d20fb0f53b204be62d125cb79083e776664" + }, + { + "path": "skills/chrome-devtools/references/puppeteer-reference.md", + "sha256": "dcec4af083b7fe2d405a40aad2b511317c1e9deaa2bec5ca91ec9959aff1a423" + }, + { + "path": "skills/chrome-devtools/references/cdp-domains.md", + "sha256": "fa81fa0fda1ae94ff2c25c924cfcb16f5a6e967b6d8412ddd372dc8a1b26f601" + }, + { + "path": "skills/chrome-devtools/references/performance-guide.md", + "sha256": "e9391edae861ecd6fac3af4384a7e692def9c9cacbb8c5d4f7ff207475f977c0" + }, + { + "path": "skills/chrome-devtools/scripts/navigate.js", + "sha256": "773cbd9a016a4db1fcaef9e0c84e8dbc4629a7207d10569d2b04ee5511ab2273" + }, + { + "path": "skills/chrome-devtools/scripts/console.js", + "sha256": "0ff057a2ac4f158668f24ccb9795d7c6d9869ca871fb5507a61092d54ff81ef5" + }, + { + "path": "skills/chrome-devtools/scripts/install.sh", + "sha256": "bdd10bfe4195ca105704c2b4ba2f641f484e3706bb6cf57030372e5a90aa10df" + }, + { + "path": "skills/chrome-devtools/scripts/click.js", + "sha256": "b81d416174532d5ec0a89a6a631550539356e67722e636da9c868c36125f88fd" + }, + { + "path": "skills/chrome-devtools/scripts/fill.js", + "sha256": "9c8b358f6ba8d1274c83ece268cb368e2f7546c001f9b9a229a80bc0e0edd516" + }, + { + "path": "skills/chrome-devtools/scripts/README.md", + "sha256": "4059687091606bace882d492073b8561eb23590994881df46f499276107e1667" + }, + { + "path": "skills/chrome-devtools/scripts/performance.js", + "sha256": "bc97995f6664336683281f47795bd8005b4bb8d8ff8ebcc952becb32cf1c668f" + }, + { + "path": "skills/chrome-devtools/scripts/screenshot.js", + "sha256": "4ec07a8c8b42ed1d7291eba0c41afd924e5c52cb2cf36a544b19375fcc6b19e7" + }, + { + "path": "skills/chrome-devtools/scripts/evaluate.js", + "sha256": "00bb01206f3a15cf33098f9ee19524fd733574bd484c99de62e9153547028d55" + }, + { + "path": "skills/chrome-devtools/scripts/package.json", + "sha256": "1ca3a64a645d7b1d2cace2256d2e3d459a6abbcfab0417429423e6fc97ce9b59" + }, + { + "path": "skills/chrome-devtools/scripts/snapshot.js", + "sha256": "d969e885d1b5a1140616282221b655117156a1bb6b4a72be9d3da02cebce04e5" + }, + { + "path": "skills/chrome-devtools/scripts/install-deps.sh", + "sha256": "c236b59df7a234079da05bdb18c22f064955a3ccea315431fa87f8a413ecc78c" + }, + { + "path": "skills/chrome-devtools/scripts/network.js", + "sha256": "492c542775a90443628ad5ae859ff556338e0b24cbd6bae23e754cca2f23b2cd" + }, + { + "path": "skills/chrome-devtools/scripts/__tests__/selector.test.js", + "sha256": "69fa6c4d04f2c8d0578325894fc06ea820e4c771d267d90969f09343e076d5e2" + }, + { + "path": "skills/chrome-devtools/scripts/lib/selector.js", + "sha256": "0c988abbec29cfb7361920c773673191a94d99c01fe1aec5a42cb48aeb6acf88" + }, + { + "path": "skills/chrome-devtools/scripts/lib/browser.js", + "sha256": "3cc9cee00475f8827b225c7f56da566a0cfb8acb8eb61557f951cb58f17f057d" + }, + { + "path": "skills/ai-multimodal/SKILL.md", + "sha256": "492e291734b1039cd7c8782fd9dc6364e7befd00fa8a26255caa8c7446146fe7" + }, + { + "path": "skills/ai-multimodal/.env.example", + "sha256": "b0786e193dfc47356939b55ae8bab12f912bf81e85bb7de9ca7155cb96a16907" + }, + { + "path": "skills/ai-multimodal/references/vision-understanding.md", + "sha256": "08de168674d3405d1c8336918a21000502cfac0ca42ba067547032fff7556afa" + }, + { + "path": "skills/ai-multimodal/references/image-generation.md", + "sha256": "1880b9e98accb7fab9b57560eeff60bc83e8519c9cee5c6192c704432f57dd8d" + }, + { + "path": "skills/ai-multimodal/references/video-analysis.md", + "sha256": "33a0df1a26b82f0f8fd9a906adfec5c969d8f792387d38cd8e89659c4ee5e4be" + }, + { + "path": "skills/ai-multimodal/references/audio-processing.md", + "sha256": "e9f9009a8ce32971470d17ecb015e37e6939d30d60acfda833afb783f044b131" + }, + { + "path": "skills/ai-multimodal/scripts/media_optimizer.py", + "sha256": "bbc3f9e1791f8cfe1bf743bcd743cf6474d8155398a7c9772fbbf5c2c6d1a6e4" + }, + { + "path": "skills/ai-multimodal/scripts/requirements.txt", + "sha256": "ca4e56c561613a6ed22b7b9b7f2853ea8f1bbe3adfc9b52103de0df7d65fae6b" + }, + { + "path": "skills/ai-multimodal/scripts/document_converter.py", + "sha256": "2067076f1476469b86c21972429643080a7a3ee6b479fb6df61285cf6269b5cb" + }, + { + "path": "skills/ai-multimodal/scripts/gemini_batch_process.py", + "sha256": "797d5670fa2a764e366a246c71b4cea8b69f4cc1725bf0411429132aff3b7b70" + }, + { + "path": "skills/ai-multimodal/scripts/tests/test_gemini_batch_process.py", + "sha256": "799bec3975ba750c056b35090148325811b1c9488ae50c05e98ac651cf9e6dfc" + }, + { + "path": "skills/ai-multimodal/scripts/tests/requirements.txt", + "sha256": "15a909ef1471e0d1fe9d523c3b02016f01abc9ab3a81b8d047a1222c3f3af2d2" + }, + { + "path": "skills/ai-multimodal/scripts/tests/.coverage", + "sha256": "22ebc0a59ff873aea8cbc28fd41651e572fdd9c5c664a0211910115679e9d79f" + }, + { + "path": "skills/ai-multimodal/scripts/tests/test_document_converter.py", + "sha256": "df6b645e5387eed2e1d1f298534870183352dde953545f254cb9957690294b52" + }, + { + "path": "skills/ai-multimodal/scripts/tests/test_media_optimizer.py", + "sha256": "8451512f789ad3519dccac1ab8221397c85188abfd528c8f9d66cecccdfd8040" + }, + { + "path": "skills/google-adk-python/SKILL.md", + "sha256": "edf5cc1d1a29fc329d425d15761e0286e431d7238c8738380181402fe09911d8" + }, + { + "path": "skills/repomix/SKILL.md", + "sha256": "0e3dafc01cef2097206eb0abee0442d5e8f8bc0307d596068950b0223320fe8f" + }, + { + "path": "skills/repomix/references/configuration.md", + "sha256": "a40386b0ca68061d03ba2ae6d1a8109306895b13699e1e05bd4b8b4c4fa7912d" + }, + { + "path": "skills/repomix/references/usage-patterns.md", + "sha256": "6e28825bf0c97e05dc367ba590487a6ecb0b3b530ff3540429d761b03b2ad1ed" + }, + { + "path": "skills/repomix/scripts/repos.example.json", + "sha256": "5086bf887ccc032abd23715c7ed278bf9844b035068b25469ed4c00435e079fd" + }, + { + "path": "skills/repomix/scripts/repomix_batch.py", + "sha256": "3b46726af95b84e0a37035f754107d2739ce1a1c95083305e166f34ab0975449" + }, + { + "path": "skills/repomix/scripts/requirements.txt", + "sha256": "abb821f8ee3248ea5c82c2d4b3c103ff411f1f44b484398186844487cf4bd9a6" + }, + { + "path": "skills/repomix/scripts/.coverage", + "sha256": "71b5da5bcfbc4f90b8cc493576c15aad70adf984b5ea7bbf5e6b294706e7ee35" + }, + { + "path": "skills/repomix/scripts/README.md", + "sha256": "0b300fe9fe67e8354085a30c7623967c444dad4bd82f5c985cecbef471fc44a4" + }, + { + "path": "skills/repomix/scripts/tests/test_repomix_batch.py", + "sha256": "4f15a013b4f1a92cc73e9fc8d1029ebb05fb8c68ac6c6a9c82d56cefd462db18" + }, + { + "path": "skills/docs-seeker/WORKFLOWS.md", + "sha256": "7ce5df192e19e0025aab94cb51e6820649df7e9e646010ba3f72f898cef8f44a" + }, + { + "path": "skills/docs-seeker/SKILL.md", + "sha256": "5f94bb0b9df6b53db457934cbb69fd440a78616e11a1e99b272f7e4849c49e7f" + }, + { + "path": "skills/docs-seeker/references/tool-selection.md", + "sha256": "ebec69a5ee56692d67544c3597ba76ce2cc787849eda634395c203e9088074e4" + }, + { + "path": "skills/docs-seeker/references/best-practices.md", + "sha256": "36f68db5da47052f5953f2a15a00f7e059440c0bd7fbc9e35df1e617ce69c14e" + }, + { + "path": "skills/docs-seeker/references/limitations.md", + "sha256": "89593bcf2e35ea022a99b141d8b54255a0ccbce29b6409e3df4d8199c65299dd" + }, + { + "path": "skills/docs-seeker/references/documentation-sources.md", + "sha256": "ad6a52d78d4157287f96d1d54c8dbd2d9a0d00e75b7453acc791f67f4697cca9" + }, + { + "path": "skills/docs-seeker/references/performance.md", + "sha256": "f9234c698dab3a666d1432ec8ac91572c259140d08e99ce5c8ace883c52a526a" + }, + { + "path": "skills/docs-seeker/references/error-handling.md", + "sha256": "0ba635c66ceb5abe172334737ee5fcd5e92cfa63f708e3951883c3e60471a0e5" + }, + { + "path": "skills/better-auth/SKILL.md", + "sha256": "bdf843e14e064b435cba53ae40e9b8d8fa8a0bbdbcb5abf5edbc8620d17822a6" + }, + { + "path": "skills/better-auth/references/database-integration.md", + "sha256": "89a873e8462dca550439e75e2dd1721835127239b5bdd71ba2cac58d35571841" + }, + { + "path": "skills/better-auth/references/email-password-auth.md", + "sha256": "a96376f55c00c2c7f0ff63bffd656fb4426a51c16c49868a7a5045b29d1bc7e3" + }, + { + "path": "skills/better-auth/references/advanced-features.md", + "sha256": "cf67bd65f28f1c09281a18c2a30e4adc425403dd688de3a3396dc8d0a5c47708" + }, + { + "path": "skills/better-auth/references/oauth-providers.md", + "sha256": "5201dc2715fe4e385e9363e67adcdeb3dd0e0879c42158e7846dd02138cf8684" + }, + { + "path": "skills/better-auth/scripts/requirements.txt", + "sha256": "4c164fd0ed4d81bba6da0852149aadfc80e169e488b86ba33cc757a3cc64bff2" + }, + { + "path": "skills/better-auth/scripts/.coverage", + "sha256": "f0eb0985b3a10c20dc53020b9aee09cbabd40e96955c47085f5678e437560528" + }, + { + "path": "skills/better-auth/scripts/better_auth_init.py", + "sha256": "82d44a3d8962186a81f7ca1615c965aae7a706332a8ab7b899d038d3c2aff69a" + }, + { + "path": "skills/better-auth/scripts/tests/test_better_auth_init.py", + "sha256": "134df79e3238f4e15e2a7c8df4d9db4c143e404a7454f8f618e56374e0de8a94" + }, + { + "path": "skills/better-auth/scripts/tests/.coverage", + "sha256": "0c77c29e26616abf909054079e5b0d0e316329e09ab0569c330e0cd2e22caa66" + }, + { + "path": "skills/web-frameworks/SKILL.md", + "sha256": "0af5655f3cb77323b3b0c6bf73ff6e9a1909ac710ab4921d306937fee964aeda" + }, + { + "path": "skills/web-frameworks/references/nextjs-data-fetching.md", + "sha256": "5175463a517033aa8d3295c4fc7420db57726017450ec637d413afcb541d3e9f" + }, + { + "path": "skills/web-frameworks/references/turborepo-setup.md", + "sha256": "429c0ed6cab365a18f3ecfb12df741ece4b4e5bc2052719b6dec97f496f81d8f" + }, + { + "path": "skills/web-frameworks/references/nextjs-optimization.md", + "sha256": "20b66c2e616ea9af62e1b3a07b68069dfc2df1574dfca6bce2e00a427ac76498" + }, + { + "path": "skills/web-frameworks/references/nextjs-server-components.md", + "sha256": "afce90a0cd5175832212bd03fdf692ee5ad7154dfce0aeb75f153fc244224801" + }, + { + "path": "skills/web-frameworks/references/turborepo-pipelines.md", + "sha256": "64279184f245524dbcaf94e29b48c56fb766e41af36e4ad92253edcc648dfe5e" + }, + { + "path": "skills/web-frameworks/references/nextjs-app-router.md", + "sha256": "bca2cc4b417f2e7aaff4a7491bdd84a10ac03ad4b42839b8c91181b5851a69c7" + }, + { + "path": "skills/web-frameworks/references/turborepo-caching.md", + "sha256": "2d7056c6cc5b5041e23c2f7bd0ff26da685c6e9513a9271803ae93bc0d2be1e2" + }, + { + "path": "skills/web-frameworks/references/remix-icon-integration.md", + "sha256": "5f2d77e5b1ddc512e2be1f270e55435e372b332e98ed56d8a399ec48226405a3" + }, + { + "path": "skills/web-frameworks/scripts/requirements.txt", + "sha256": "89002a89d32b1f2e71cfb62fe48fe52580aedd6001a4fe761fd1edb8f727af28" + }, + { + "path": "skills/web-frameworks/scripts/turborepo_migrate.py", + "sha256": "d31252fe5cf3fe7ed89ded47e8f0b76ac945a84cec2c7ae77c964b4e2c7934a1" + }, + { + "path": "skills/web-frameworks/scripts/.coverage", + "sha256": "b27f663f8575c76587c3073b7aab28e6e5d26af02d95abfd55417e142e7268bf" + }, + { + "path": "skills/web-frameworks/scripts/__init__.py", + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + { + "path": "skills/web-frameworks/scripts/nextjs_init.py", + "sha256": "8aefe43870792d3690434d56db6ee927d7785302e3e98ff4abbb0b80f3a54b9c" + }, + { + "path": "skills/web-frameworks/scripts/tests/requirements.txt", + "sha256": "0795bdcfb80afae0ff06e9cbe5bed67b39283e577984bed87d85da3fec798a37" + }, + { + "path": "skills/web-frameworks/scripts/tests/test_turborepo_migrate.py", + "sha256": "2897b6b62df7a8b7d029520ac32a92634e6a19d3cac043ff4ffc3441c014c671" + }, + { + "path": "skills/web-frameworks/scripts/tests/test_nextjs_init.py", + "sha256": "770ea000b05f56cf199aa93533a070afd2f6113dd1f7a975de0a5ef0168f64d9" + }, + { + "path": "skills/web-frameworks/scripts/tests/coverage-web.json", + "sha256": "737e65f4900cc685612f9831ebd622567efb3bd7de55dd9d97b5aa2e04cd89ec" + }, + { + "path": "skills/mcp-builder/SKILL.md", + "sha256": "b1010e90adcb8fd6bf57640df34ab6454fbf7e4216e150a4620f7caccadc4e63" + }, + { + "path": "skills/mcp-builder/LICENSE.txt", + "sha256": "58d1e17ffe5109a7ae296caafcadfdbe6a7d176f0bc4ab01e12a689b0499d8bd" + }, + { + "path": "skills/mcp-builder/scripts/requirements.txt", + "sha256": "d5d7558b2368ecea9dfeed7d1fbc71ee9e0750bebd1282faa527d528a344c3c7" + }, + { + "path": "skills/mcp-builder/scripts/evaluation.py", + "sha256": "49ed1d17cdce5da101b210197740713f49b935c29d4f339542a14b132658e6f7" + }, + { + "path": "skills/mcp-builder/scripts/connections.py", + "sha256": "9403668a2041568772082a8b334122c1f88daf0541fb393af4522d0094a47a6e" + }, + { + "path": "skills/mcp-builder/scripts/example_evaluation.xml", + "sha256": "9272b348ddcc4b06ba562367ccd0770e018158c0068ac5116d5e34aaeff8777a" + }, + { + "path": "skills/mcp-builder/reference/mcp_best_practices.md", + "sha256": "3bdf013379bdd3c198baccd0f183441c710fc7cae07ba4c6f8f8048276519688" + }, + { + "path": "skills/mcp-builder/reference/python_mcp_server.md", + "sha256": "4e6db48188f44ff4eb707f50b8d273d5d18af4b88d326f7a26f03a405064bc0b" + }, + { + "path": "skills/mcp-builder/reference/node_mcp_server.md", + "sha256": "40b03e9c07463d5db524c1f5140ef60713fdd911c2f4386f89e0b94d43b8764e" + }, + { + "path": "skills/mcp-builder/reference/evaluation.md", + "sha256": "8c99479f8a2d22a636c38e274537aac3610879e26f34e0709825077c4576f427" + }, + { + "path": "skills/sequential-thinking/README.md", + "sha256": "3571f9547aa31d7b316caa4dbc636ce9dba03ffacd73f2d3d9eef57bb3de6f22" + }, + { + "path": "skills/sequential-thinking/SKILL.md", + "sha256": "c517ae710853ee2ea06111d72888fbe1eb432368b0a7984b9c31391ecbc2ef27" + }, + { + "path": "skills/sequential-thinking/references/examples.md", + "sha256": "f0bd40121128acf978ebbaa5761c79efbfe3ca11e49a473ef62d8d7bd0f58f8f" + }, + { + "path": "skills/sequential-thinking/references/advanced.md", + "sha256": "94a5fe40dd7150612374578b389a8428c3e69cc02e838007185a0d60aee75cdf" + } + ], + "dirSha256": "67012aff99a0fb8498e5a2234fd4ab558e85c60d85822d5ab8a35ad610581bf2" + }, + "security": { + "scannedAt": null, + "scannerVersion": null, + "flags": [] + } +} \ No newline at end of file diff --git a/skills/ai-multimodal/.env.example b/skills/ai-multimodal/.env.example new file mode 100644 index 0000000..53663c4 --- /dev/null +++ b/skills/ai-multimodal/.env.example @@ -0,0 +1,97 @@ +# Google Gemini API Configuration + +# ============================================================================ +# OPTION 1: Google AI Studio (Default - Recommended for most users) +# ============================================================================ +# Get your API key: https://aistudio.google.com/apikey +GEMINI_API_KEY=your_api_key_here + +# ============================================================================ +# OPTION 2: Vertex AI (Google Cloud Platform) +# ============================================================================ +# Uncomment these lines to use Vertex AI instead of Google AI Studio +# GEMINI_USE_VERTEX=true +# VERTEX_PROJECT_ID=your-gcp-project-id +# VERTEX_LOCATION=us-central1 + +# ============================================================================ +# Model Selection (Optional) +# ============================================================================ +# Override default model for specific tasks +# Default: gemini-2.5-flash for most tasks +# GEMINI_MODEL=gemini-2.5-flash +# GEMINI_IMAGE_GEN_MODEL=gemini-2.5-flash-image + +# ============================================================================ +# Rate Limiting Configuration (Optional) +# ============================================================================ +# Requests per minute limit (adjust based on your tier) +# GEMINI_RPM_LIMIT=15 + +# Tokens per minute limit +# GEMINI_TPM_LIMIT=4000000 + +# Requests per day limit +# GEMINI_RPD_LIMIT=1500 + +# ============================================================================ +# Processing Options (Optional) +# ============================================================================ +# Video resolution mode: default or low-res +# low-res uses ~100 tokens/second vs ~300 for default +# GEMINI_VIDEO_RESOLUTION=default + +# Audio quality: default (16 Kbps mono, auto-downsampled) +# GEMINI_AUDIO_QUALITY=default + +# PDF processing mode: inline (<20MB) or file-api (>20MB, automatic) +# GEMINI_PDF_MODE=auto + +# ============================================================================ +# Retry Configuration (Optional) +# ============================================================================ +# Maximum retry attempts for failed requests +# GEMINI_MAX_RETRIES=3 + +# Initial retry delay in seconds (uses exponential backoff) +# GEMINI_RETRY_DELAY=1 + +# ============================================================================ +# Output Configuration (Optional) +# ============================================================================ +# Default output directory for generated images +# OUTPUT_DIR=./output + +# Image output format (png or jpeg) +# IMAGE_FORMAT=png + +# Image quality for JPEG (1-100) +# IMAGE_QUALITY=95 + +# ============================================================================ +# Context Caching (Optional) +# ============================================================================ +# Enable context caching for repeated queries on same file +# GEMINI_ENABLE_CACHING=true + +# Cache TTL in seconds (default: 1800 = 30 minutes) +# GEMINI_CACHE_TTL=1800 + +# ============================================================================ +# Logging (Optional) +# ============================================================================ +# Log level: DEBUG, INFO, WARNING, ERROR, CRITICAL +# LOG_LEVEL=INFO + +# Log file path +# LOG_FILE=./logs/gemini.log + +# ============================================================================ +# Notes +# ============================================================================ +# 1. Never commit API keys to version control +# 2. Add .env to .gitignore +# 3. API keys can be restricted in Google Cloud Console +# 4. Monitor usage at: https://aistudio.google.com/apikey +# 5. Free tier limits: 15 RPM, 1M-4M TPM, 1,500 RPD +# 6. Vertex AI requires GCP authentication via gcloud CLI diff --git a/skills/ai-multimodal/SKILL.md b/skills/ai-multimodal/SKILL.md new file mode 100644 index 0000000..aa33d45 --- /dev/null +++ b/skills/ai-multimodal/SKILL.md @@ -0,0 +1,357 @@ +--- +name: ai-multimodal +description: Process and generate multimedia content using Google Gemini API. Capabilities include analyze audio files (transcription with timestamps, summarization, speech understanding, music/sound analysis up to 9.5 hours), understand images (captioning, object detection, OCR, visual Q&A, segmentation), process videos (scene detection, Q&A, temporal analysis, YouTube URLs, up to 6 hours), extract from documents (PDF tables, forms, charts, diagrams, multi-page), generate images (text-to-image, editing, composition, refinement). Use when working with audio/video files, analyzing images or screenshots, processing PDF documents, extracting structured data from media, creating images from text prompts, or implementing multimodal AI features. Supports multiple models (Gemini 2.5/2.0) with context windows up to 2M tokens. +license: MIT +allowed-tools: + - Bash + - Read + - Write + - Edit +--- + +# AI Multimodal Processing Skill + +Process audio, images, videos, documents, and generate images using Google Gemini's multimodal API. Unified interface for all multimedia content understanding and generation. + +## Core Capabilities + +### Audio Processing +- Transcription with timestamps (up to 9.5 hours) +- Audio summarization and analysis +- Speech understanding and speaker identification +- Music and environmental sound analysis +- Text-to-speech generation with controllable voice + +### Image Understanding +- Image captioning and description +- Object detection with bounding boxes (2.0+) +- Pixel-level segmentation (2.5+) +- Visual question answering +- Multi-image comparison (up to 3,600 images) +- OCR and text extraction + +### Video Analysis +- Scene detection and summarization +- Video Q&A with temporal understanding +- Transcription with visual descriptions +- YouTube URL support +- Long video processing (up to 6 hours) +- Frame-level analysis + +### Document Extraction +- Native PDF vision processing (up to 1,000 pages) +- Table and form extraction +- Chart and diagram analysis +- Multi-page document understanding +- Structured data output (JSON schema) +- Format conversion (PDF to HTML/JSON) + +### Image Generation +- Text-to-image generation +- Image editing and modification +- Multi-image composition (up to 3 images) +- Iterative refinement +- Multiple aspect ratios (1:1, 16:9, 9:16, 4:3, 3:4) +- Controllable style and quality + +## Capability Matrix + +| Task | Audio | Image | Video | Document | Generation | +|------|:-----:|:-----:|:-----:|:--------:|:----------:| +| Transcription | ✓ | - | ✓ | - | - | +| Summarization | ✓ | ✓ | ✓ | ✓ | - | +| Q&A | ✓ | ✓ | ✓ | ✓ | - | +| Object Detection | - | ✓ | ✓ | - | - | +| Text Extraction | - | ✓ | - | ✓ | - | +| Structured Output | ✓ | ✓ | ✓ | ✓ | - | +| Creation | TTS | - | - | - | ✓ | +| Timestamps | ✓ | - | ✓ | - | - | +| Segmentation | - | ✓ | - | - | - | + +## Model Selection Guide + +### Gemini 2.5 Series (Recommended) +- **gemini-2.5-pro**: Highest quality, all features, 1M-2M context +- **gemini-2.5-flash**: Best balance, all features, 1M-2M context +- **gemini-2.5-flash-lite**: Lightweight, segmentation support +- **gemini-2.5-flash-image**: Image generation only + +### Gemini 2.0 Series +- **gemini-2.0-flash**: Fast processing, object detection +- **gemini-2.0-flash-lite**: Lightweight option + +### Feature Requirements +- **Segmentation**: Requires 2.5+ models +- **Object Detection**: Requires 2.0+ models +- **Multi-video**: Requires 2.5+ models +- **Image Generation**: Requires flash-image model + +### Context Windows +- **2M tokens**: ~6 hours video (low-res) or ~2 hours (default) +- **1M tokens**: ~3 hours video (low-res) or ~1 hour (default) +- **Audio**: 32 tokens/second (1 min = 1,920 tokens) +- **PDF**: 258 tokens/page (fixed) +- **Image**: 258-1,548 tokens based on size + +## Quick Start + +### Prerequisites + +**API Key Setup**: Supports both Google AI Studio and Vertex AI. + +The skill checks for `GEMINI_API_KEY` in this order: +1. Process environment: `export GEMINI_API_KEY="your-key"` +2. Project root: `.env` +3. `.claude/.env` +4. `.claude/skills/.env` +5. `.claude/skills/ai-multimodal/.env` + +**Get API key**: https://aistudio.google.com/apikey + +**For Vertex AI**: +```bash +export GEMINI_USE_VERTEX=true +export VERTEX_PROJECT_ID=your-gcp-project-id +export VERTEX_LOCATION=us-central1 # Optional +``` + +**Install SDK**: +```bash +pip install google-genai python-dotenv pillow +``` + +### Common Patterns + +**Transcribe Audio**: +```bash +python scripts/gemini_batch_process.py \ + --files audio.mp3 \ + --task transcribe \ + --model gemini-2.5-flash +``` + +**Analyze Image**: +```bash +python scripts/gemini_batch_process.py \ + --files image.jpg \ + --task analyze \ + --prompt "Describe this image" \ + --output docs/assets/.md \ + --model gemini-2.5-flash +``` + +**Process Video**: +```bash +python scripts/gemini_batch_process.py \ + --files video.mp4 \ + --task analyze \ + --prompt "Summarize key points with timestamps" \ + --output docs/assets/.md \ + --model gemini-2.5-flash +``` + +**Extract from PDF**: +```bash +python scripts/gemini_batch_process.py \ + --files document.pdf \ + --task extract \ + --prompt "Extract table data as JSON" \ + --output docs/assets/.md \ + --format json +``` + +**Generate Image**: +```bash +python scripts/gemini_batch_process.py \ + --task generate \ + --prompt "A futuristic city at sunset" \ + --output docs/assets/ \ + --model gemini-2.5-flash-image \ + --aspect-ratio 16:9 +``` + +**Optimize Media**: +```bash +# Prepare large video for processing +python scripts/media_optimizer.py \ + --input large-video.mp4 \ + --output docs/assets/ \ + --target-size 100MB + +# Batch optimize multiple files +python scripts/media_optimizer.py \ + --input-dir ./videos \ + --output-dir docs/assets/optimized \ + --quality 85 +``` + +**Convert Documents to Markdown**: +```bash +# Convert to PDF +python scripts/document_converter.py \ + --input document.docx \ + --output docs/assets/document.md + +# Extract pages +python scripts/document_converter.py \ + --input large.pdf \ + --output docs/assets/chapter1.md \ + --pages 1-20 +``` + +## Supported Formats + +### Audio +- WAV, MP3, AAC, FLAC, OGG Vorbis, AIFF +- Max 9.5 hours per request +- Auto-downsampled to 16 Kbps mono + +### Images +- PNG, JPEG, WEBP, HEIC, HEIF +- Max 3,600 images per request +- Resolution: ≤384px = 258 tokens, larger = tiled + +### Video +- MP4, MPEG, MOV, AVI, FLV, MPG, WebM, WMV, 3GPP +- Max 6 hours (low-res) or 2 hours (default) +- YouTube URLs supported (public only) + +### Documents +- PDF only for vision processing +- Max 1,000 pages +- TXT, HTML, Markdown supported (text-only) + +### Size Limits +- **Inline**: <20MB total request +- **File API**: 2GB per file, 20GB project quota +- **Retention**: 48 hours auto-delete + +## Reference Navigation + +For detailed implementation guidance, see: + +### Audio Processing +- `references/audio-processing.md` - Transcription, analysis, TTS + - Timestamp handling and segment analysis + - Multi-speaker identification + - Non-speech audio analysis + - Text-to-speech generation + +### Image Understanding +- `references/vision-understanding.md` - Captioning, detection, OCR + - Object detection and localization + - Pixel-level segmentation + - Visual question answering + - Multi-image comparison + +### Video Analysis +- `references/video-analysis.md` - Scene detection, temporal understanding + - YouTube URL processing + - Timestamp-based queries + - Video clipping and FPS control + - Long video optimization + +### Document Extraction +- `references/document-extraction.md` - PDF processing, structured output + - Table and form extraction + - Chart and diagram analysis + - JSON schema validation + - Multi-page handling + +### Image Generation +- `references/image-generation.md` - Text-to-image, editing + - Prompt engineering strategies + - Image editing and composition + - Aspect ratio selection + - Safety settings + +## Cost Optimization + +### Token Costs +**Input Pricing**: +- Gemini 2.5 Flash: $1.00/1M input, $0.10/1M output +- Gemini 2.5 Pro: $3.00/1M input, $12.00/1M output +- Gemini 1.5 Flash: $0.70/1M input, $0.175/1M output + +**Token Rates**: +- Audio: 32 tokens/second (1 min = 1,920 tokens) +- Video: ~300 tokens/second (default) or ~100 (low-res) +- PDF: 258 tokens/page (fixed) +- Image: 258-1,548 tokens based on size + +**TTS Pricing**: +- Flash TTS: $10/1M tokens +- Pro TTS: $20/1M tokens + +### Best Practices +1. Use `gemini-2.5-flash` for most tasks (best price/performance) +2. Use File API for files >20MB or repeated queries +3. Optimize media before upload (see `media_optimizer.py`) +4. Process specific segments instead of full videos +5. Use lower FPS for static content +6. Implement context caching for repeated queries +7. Batch process multiple files in parallel + +## Rate Limits + +**Free Tier**: +- 10-15 RPM (requests per minute) +- 1M-4M TPM (tokens per minute) +- 1,500 RPD (requests per day) + +**YouTube Limits**: +- Free tier: 8 hours/day +- Paid tier: No length limits +- Public videos only + +**Storage Limits**: +- 20GB per project +- 2GB per file +- 48-hour retention + +## Error Handling + +Common errors and solutions: +- **400**: Invalid format/size - validate before upload +- **401**: Invalid API key - check configuration +- **403**: Permission denied - verify API key restrictions +- **404**: File not found - ensure file uploaded and active +- **429**: Rate limit exceeded - implement exponential backoff +- **500**: Server error - retry with backoff + +## Scripts Overview + +All scripts support unified API key detection and error handling: + +**gemini_batch_process.py**: Batch process multiple media files +- Supports all modalities (audio, image, video, PDF) +- Progress tracking and error recovery +- Output formats: JSON, Markdown, CSV +- Rate limiting and retry logic +- Dry-run mode + +**media_optimizer.py**: Prepare media for Gemini API +- Compress videos/audio for size limits +- Resize images appropriately +- Split long videos into chunks +- Format conversion +- Quality vs size optimization + +**document_converter.py**: Convert documents to PDF +- Convert DOCX, XLSX, PPTX to PDF +- Extract page ranges +- Optimize PDFs for Gemini +- Extract images from PDFs +- Batch conversion support + +Run any script with `--help` for detailed usage. + +## Resources + +- [Audio API Docs](https://ai.google.dev/gemini-api/docs/audio) +- [Image API Docs](https://ai.google.dev/gemini-api/docs/image-understanding) +- [Video API Docs](https://ai.google.dev/gemini-api/docs/video-understanding) +- [Document API Docs](https://ai.google.dev/gemini-api/docs/document-processing) +- [Image Gen Docs](https://ai.google.dev/gemini-api/docs/image-generation) +- [Get API Key](https://aistudio.google.com/apikey) +- [Pricing](https://ai.google.dev/pricing) diff --git a/skills/ai-multimodal/references/audio-processing.md b/skills/ai-multimodal/references/audio-processing.md new file mode 100644 index 0000000..b99c889 --- /dev/null +++ b/skills/ai-multimodal/references/audio-processing.md @@ -0,0 +1,373 @@ +# Audio Processing Reference + +Comprehensive guide for audio analysis and speech generation using Gemini API. + +## Audio Understanding + +### Supported Formats + +| Format | MIME Type | Best Use | +|--------|-----------|----------| +| WAV | `audio/wav` | Uncompressed, highest quality | +| MP3 | `audio/mp3` | Compressed, widely compatible | +| AAC | `audio/aac` | Compressed, good quality | +| FLAC | `audio/flac` | Lossless compression | +| OGG Vorbis | `audio/ogg` | Open format | +| AIFF | `audio/aiff` | Apple format | + +### Specifications + +- **Maximum length**: 9.5 hours per request +- **Multiple files**: Unlimited count, combined max 9.5 hours +- **Token rate**: 32 tokens/second (1 minute = 1,920 tokens) +- **Processing**: Auto-downsampled to 16 Kbps mono +- **File size limits**: + - Inline: 20 MB max total request + - File API: 2 GB per file, 20 GB project quota + - Retention: 48 hours auto-delete + +## Transcription + +### Basic Transcription + +```python +from google import genai +import os + +client = genai.Client(api_key=os.getenv('GEMINI_API_KEY')) + +# Upload audio +myfile = client.files.upload(file='meeting.mp3') + +# Transcribe +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Generate a transcript of the speech.', myfile] +) +print(response.text) +``` + +### With Timestamps + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Generate transcript with timestamps in MM:SS format.', myfile] +) +``` + +### Multi-Speaker Identification + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Transcribe with speaker labels. Format: [Speaker 1], [Speaker 2], etc.', myfile] +) +``` + +### Segment-Specific Transcription + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Transcribe only the segment from 02:30 to 05:15.', myfile] +) +``` + +## Audio Analysis + +### Summarization + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Summarize key points in 5 bullets with timestamps.', myfile] +) +``` + +### Non-Speech Audio Analysis + +```python +# Music analysis +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Identify the musical instruments and genre.', myfile] +) + +# Environmental sounds +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Identify all sounds: voices, music, ambient noise.', myfile] +) + +# Birdsong identification +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Identify bird species based on their calls.', myfile] +) +``` + +### Timestamp-Based Analysis + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['What is discussed from 10:30 to 15:45? Provide key points.', myfile] +) +``` + +## Input Methods + +### File Upload (>20MB or Reuse) + +```python +# Upload once, use multiple times +myfile = client.files.upload(file='large-audio.mp3') + +# First query +response1 = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Transcribe this', myfile] +) + +# Second query (reuses same file) +response2 = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Summarize this', myfile] +) +``` + +### Inline Data (<20MB) + +```python +from google.genai import types + +with open('small-audio.mp3', 'rb') as f: + audio_bytes = f.read() + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Describe this audio', + types.Part.from_bytes(data=audio_bytes, mime_type='audio/mp3') + ] +) +``` + +## Speech Generation (TTS) + +### Available Models + +| Model | Quality | Speed | Cost/1M tokens | +|-------|---------|-------|----------------| +| `gemini-2.5-flash-native-audio-preview-09-2025` | High | Fast | $10 | +| `gemini-2.5-pro` TTS mode | Premium | Slower | $20 | + +### Basic TTS + +```python +response = client.models.generate_content( + model='gemini-2.5-flash-native-audio-preview-09-2025', + contents='Generate audio: Welcome to today\'s episode.' +) + +# Save audio +with open('output.wav', 'wb') as f: + f.write(response.audio_data) +``` + +### Controllable Voice Style + +```python +# Professional tone +response = client.models.generate_content( + model='gemini-2.5-flash-native-audio-preview-09-2025', + contents='Generate audio in a professional, clear tone: Welcome to our quarterly earnings call.' +) + +# Casual and friendly +response = client.models.generate_content( + model='gemini-2.5-flash-native-audio-preview-09-2025', + contents='Generate audio in a friendly, conversational tone: Hey there! Let\'s dive into today\'s topic.' +) + +# Narrative style +response = client.models.generate_content( + model='gemini-2.5-flash-native-audio-preview-09-2025', + contents='Generate audio in a narrative, storytelling tone: Once upon a time, in a land far away...' +) +``` + +### Voice Control Parameters + +- **Style**: Professional, casual, narrative, conversational +- **Pace**: Slow, normal, fast +- **Tone**: Friendly, serious, enthusiastic +- **Accent**: Natural language control (e.g., "British accent", "Southern drawl") + +## Best Practices + +### File Management + +1. Use File API for files >20MB +2. Use File API for repeated queries (saves tokens) +3. Files auto-delete after 48 hours +4. Clean up manually when done: + ```python + client.files.delete(name=myfile.name) + ``` + +### Prompt Engineering + +**Effective prompts**: +- "Transcribe from 02:30 to 03:29 in MM:SS format" +- "Identify speakers and extract dialogue with timestamps" +- "Summarize key points with relevant timestamps" +- "Transcribe and analyze sentiment for each speaker" + +**Context improves accuracy**: +- "This is a medical interview - use appropriate terminology" +- "Transcribe this legal deposition with precise terminology" +- "This is a technical podcast about machine learning" + +**Combined tasks**: +- "Transcribe and summarize in bullet points" +- "Extract key quotes with timestamps and speaker labels" +- "Transcribe and identify action items with timestamps" + +### Cost Optimization + +**Token calculation**: +- 1 minute audio = 1,920 tokens +- 1 hour audio = 115,200 tokens +- 9.5 hours = 1,094,400 tokens + +**Model selection**: +- Use `gemini-2.5-flash` ($1/1M tokens) for most tasks +- Upgrade to `gemini-2.5-pro` ($3/1M tokens) for complex analysis +- For high-volume: `gemini-1.5-flash` ($0.70/1M tokens) + +**Reduce costs**: +- Process only relevant segments using timestamps +- Use lower-quality audio when possible +- Batch multiple short files in one request +- Cache context for repeated queries + +### Error Handling + +```python +import time + +def transcribe_with_retry(file_path, max_retries=3): + """Transcribe audio with exponential backoff retry""" + for attempt in range(max_retries): + try: + myfile = client.files.upload(file=file_path) + response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Transcribe with timestamps', myfile] + ) + return response.text + except Exception as e: + if attempt == max_retries - 1: + raise + wait_time = 2 ** attempt + print(f"Retry {attempt + 1} after {wait_time}s") + time.sleep(wait_time) +``` + +## Common Use Cases + +### 1. Meeting Transcription + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Transcribe this meeting with: + 1. Speaker labels + 2. Timestamps for topic changes + 3. Action items highlighted + ''', + myfile + ] +) +``` + +### 2. Podcast Summary + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Create podcast summary with: + 1. Main topics with timestamps + 2. Key quotes from each speaker + 3. Recommended episode highlights + ''', + myfile + ] +) +``` + +### 3. Interview Analysis + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Analyze interview: + 1. Questions asked with timestamps + 2. Key responses from interviewee + 3. Overall sentiment and tone + ''', + myfile + ] +) +``` + +### 4. Content Verification + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Verify audio content: + 1. Check for specific keywords or phrases + 2. Identify any compliance issues + 3. Note any concerning statements with timestamps + ''', + myfile + ] +) +``` + +### 5. Multilingual Transcription + +```python +# Gemini auto-detects language +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Transcribe this audio and translate to English if needed.', myfile] +) +``` + +## Token Costs + +**Audio Input** (32 tokens/second): +- 1 minute = 1,920 tokens +- 10 minutes = 19,200 tokens +- 1 hour = 115,200 tokens +- 9.5 hours = 1,094,400 tokens + +**Example costs** (Gemini 2.5 Flash at $1/1M): +- 1 hour audio: 115,200 tokens = $0.12 +- Full day podcast (8 hours): 921,600 tokens = $0.92 + +## Limitations + +- Maximum 9.5 hours per request +- Auto-downsampled to 16 Kbps mono (quality loss) +- Files expire after 48 hours +- No real-time streaming support +- Non-speech audio less accurate than speech diff --git a/skills/ai-multimodal/references/image-generation.md b/skills/ai-multimodal/references/image-generation.md new file mode 100644 index 0000000..df18b7e --- /dev/null +++ b/skills/ai-multimodal/references/image-generation.md @@ -0,0 +1,558 @@ +# Image Generation Reference + +Comprehensive guide for image creation, editing, and composition using Gemini API. + +## Core Capabilities + +- **Text-to-Image**: Generate images from text prompts +- **Image Editing**: Modify existing images with text instructions +- **Multi-Image Composition**: Combine up to 3 images +- **Iterative Refinement**: Refine images conversationally +- **Aspect Ratios**: Multiple formats (1:1, 16:9, 9:16, 4:3, 3:4) +- **Style Control**: Control artistic style and quality +- **Text in Images**: Limited text rendering (max 25 chars) + +## Model + +**gemini-2.5-flash-image** - Specialized for image generation +- Input tokens: 65,536 +- Output tokens: 32,768 +- Knowledge cutoff: June 2025 +- Supports: Text and image inputs, image outputs + +## Quick Start + +### Basic Generation + +```python +from google import genai +from google.genai import types +import os + +client = genai.Client(api_key=os.getenv('GEMINI_API_KEY')) + +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='A serene mountain landscape at sunset with snow-capped peaks', + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='16:9' + ) +) + +# Save image +for i, part in enumerate(response.candidates[0].content.parts): + if part.inline_data: + with open(f'output-{i}.png', 'wb') as f: + f.write(part.inline_data.data) +``` + +## Aspect Ratios + +| Ratio | Resolution | Use Case | Token Cost | +|-------|-----------|----------|------------| +| 1:1 | 1024×1024 | Social media, avatars | 1290 | +| 16:9 | 1344×768 | Landscapes, banners | 1290 | +| 9:16 | 768×1344 | Mobile, portraits | 1290 | +| 4:3 | 1152×896 | Traditional media | 1290 | +| 3:4 | 896×1152 | Vertical posters | 1290 | + +All ratios cost the same: 1,290 tokens per image. + +## Response Modalities + +### Image Only + +```python +config = types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='1:1' +) +``` + +### Text Only (No Image) + +```python +config = types.GenerateContentConfig( + response_modalities=['text'] +) +# Returns text description instead of generating image +``` + +### Both Image and Text + +```python +config = types.GenerateContentConfig( + response_modalities=['image', 'text'], + aspect_ratio='16:9' +) +# Returns both generated image and description +``` + +## Image Editing + +### Modify Existing Image + +```python +import PIL.Image + +# Load original +img = PIL.Image.open('original.png') + +# Edit with instructions +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=[ + 'Add a red balloon floating in the sky', + img + ], + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='16:9' + ) +) +``` + +### Style Transfer + +```python +img = PIL.Image.open('photo.jpg') + +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=[ + 'Transform this into an oil painting style', + img + ] +) +``` + +### Object Addition/Removal + +```python +# Add object +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=[ + 'Add a vintage car parked on the street', + img + ] +) + +# Remove object +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=[ + 'Remove the person on the left side', + img + ] +) +``` + +## Multi-Image Composition + +### Combine Multiple Images + +```python +img1 = PIL.Image.open('background.png') +img2 = PIL.Image.open('foreground.png') +img3 = PIL.Image.open('overlay.png') + +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=[ + 'Combine these images into a cohesive scene', + img1, + img2, + img3 + ], + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='16:9' + ) +) +``` + +**Note**: Recommended maximum 3 input images for best results. + +## Prompt Engineering + +### Effective Prompt Structure + +**Three key elements**: +1. **Subject**: What to generate +2. **Context**: Environmental setting +3. **Style**: Artistic treatment + +**Example**: "A robot [subject] in a futuristic city [context], cyberpunk style with neon lighting [style]" + +### Quality Modifiers + +**Technical terms**: +- "4K", "8K", "high resolution" +- "HDR", "high dynamic range" +- "professional photography" +- "studio lighting" +- "ultra detailed" + +**Camera settings**: +- "35mm lens", "50mm lens" +- "shallow depth of field" +- "wide angle shot" +- "macro photography" +- "golden hour lighting" + +### Style Keywords + +**Art styles**: +- "oil painting", "watercolor", "sketch" +- "digital art", "concept art" +- "photorealistic", "hyperrealistic" +- "minimalist", "abstract" +- "cyberpunk", "steampunk", "fantasy" + +**Mood and atmosphere**: +- "dramatic lighting", "soft lighting" +- "moody", "bright and cheerful" +- "mysterious", "whimsical" +- "dark and gritty", "pastel colors" + +### Subject Description + +**Be specific**: +- ❌ "A cat" +- ✅ "A fluffy orange tabby cat with green eyes" + +**Add context**: +- ❌ "A building" +- ✅ "A modern glass skyscraper reflecting sunset clouds" + +**Include details**: +- ❌ "A person" +- ✅ "A young woman in a red dress holding an umbrella" + +### Composition and Framing + +**Camera angles**: +- "bird's eye view", "aerial shot" +- "low angle", "high angle" +- "close-up", "wide shot" +- "centered composition" +- "rule of thirds" + +**Perspective**: +- "first person view" +- "third person perspective" +- "isometric view" +- "forced perspective" + +### Text in Images + +**Limitations**: +- Maximum 25 characters total +- Up to 3 distinct text phrases +- Works best with simple text + +**Best practices**: +```python +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='A vintage poster with bold text "EXPLORE" at the top, mountain landscape, retro 1950s style' +) +``` + +**Font control**: +- "bold sans-serif title" +- "handwritten script" +- "vintage letterpress" +- "modern minimalist font" + +## Advanced Techniques + +### Iterative Refinement + +```python +# Initial generation +response1 = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='A futuristic city skyline' +) + +# Save first version +with open('v1.png', 'wb') as f: + f.write(response1.candidates[0].content.parts[0].inline_data.data) + +# Refine +img = PIL.Image.open('v1.png') +response2 = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=[ + 'Add flying vehicles and neon signs', + img + ] +) +``` + +### Negative Prompts (Indirect) + +```python +# Instead of "no blur", be specific about what you want +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='A crystal clear, sharp photograph of a diamond ring with perfect focus and high detail' +) +``` + +### Consistent Style Across Images + +```python +base_prompt = "Digital art, vibrant colors, cel-shaded style, clean lines" + +prompts = [ + f"{base_prompt}, a warrior character", + f"{base_prompt}, a mage character", + f"{base_prompt}, a rogue character" +] + +for i, prompt in enumerate(prompts): + response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=prompt + ) + # Save each character +``` + +## Safety Settings + +### Configure Safety Filters + +```python +config = types.GenerateContentConfig( + response_modalities=['image'], + safety_settings=[ + types.SafetySetting( + category=types.HarmCategory.HARM_CATEGORY_HATE_SPEECH, + threshold=types.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE + ), + types.SafetySetting( + category=types.HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT, + threshold=types.HarmBlockThreshold.BLOCK_MEDIUM_AND_ABOVE + ) + ] +) +``` + +### Available Categories + +- `HARM_CATEGORY_HATE_SPEECH` +- `HARM_CATEGORY_DANGEROUS_CONTENT` +- `HARM_CATEGORY_HARASSMENT` +- `HARM_CATEGORY_SEXUALLY_EXPLICIT` + +### Thresholds + +- `BLOCK_NONE`: No blocking +- `BLOCK_LOW_AND_ABOVE`: Block low probability and above +- `BLOCK_MEDIUM_AND_ABOVE`: Block medium and above (default) +- `BLOCK_ONLY_HIGH`: Block only high probability + +## Common Use Cases + +### 1. Marketing Assets + +```python +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='''Professional product photography: + - Sleek smartphone on minimalist white surface + - Dramatic side lighting creating subtle shadows + - Shallow depth of field, crisp focus + - Clean, modern aesthetic + - 4K quality + ''', + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='4:3' + ) +) +``` + +### 2. Concept Art + +```python +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='''Fantasy concept art: + - Ancient floating islands connected by chains + - Waterfalls cascading into clouds below + - Magical crystals glowing on the islands + - Epic scale, dramatic lighting + - Detailed digital painting style + ''', + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='16:9' + ) +) +``` + +### 3. Social Media Graphics + +```python +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='''Instagram post design: + - Pastel gradient background (pink to blue) + - Motivational quote layout + - Modern minimalist style + - Clean typography + - Mobile-friendly composition + ''', + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='1:1' + ) +) +``` + +### 4. Illustration + +```python +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='''Children's book illustration: + - Friendly cartoon dragon reading a book + - Bright, cheerful colors + - Soft, rounded shapes + - Whimsical forest background + - Warm, inviting atmosphere + ''', + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='4:3' + ) +) +``` + +### 5. UI/UX Mockups + +```python +response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents='''Modern mobile app interface: + - Clean dashboard design + - Card-based layout + - Soft shadows and gradients + - Contemporary color scheme (blue and white) + - Professional fintech aesthetic + ''', + config=types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='9:16' + ) +) +``` + +## Best Practices + +### Prompt Quality + +1. **Be specific**: More detail = better results +2. **Order matters**: Most important elements first +3. **Use examples**: Reference known styles or artists +4. **Avoid contradictions**: Don't ask for opposing styles +5. **Test and iterate**: Refine prompts based on results + +### File Management + +```python +# Save with descriptive names +timestamp = int(time.time()) +filename = f'generated_{timestamp}_{aspect_ratio}.png' + +with open(filename, 'wb') as f: + f.write(image_data) +``` + +### Cost Optimization + +**Token costs**: +- 1 image: 1,290 tokens = $0.00129 (Flash Image at $1/1M) +- 10 images: 12,900 tokens = $0.0129 +- 100 images: 129,000 tokens = $0.129 + +**Strategies**: +- Generate fewer iterations +- Use text modality first to validate concept +- Batch similar requests +- Cache prompts for consistent style + +## Error Handling + +### Safety Filter Blocking + +```python +try: + response = client.models.generate_content( + model='gemini-2.5-flash-image', + contents=prompt + ) +except Exception as e: + # Check block reason + if hasattr(e, 'prompt_feedback'): + print(f"Blocked: {e.prompt_feedback.block_reason}") + # Modify prompt and retry +``` + +### Token Limit Exceeded + +```python +# Keep prompts concise +if len(prompt) > 1000: + # Truncate or simplify + prompt = prompt[:1000] +``` + +## Limitations + +- Maximum 3 input images for composition +- Text rendering limited (25 chars max) +- No video or animation generation +- Regional restrictions (child images in EEA, CH, UK) +- Optimal language support: English, Spanish (Mexico), Japanese, Mandarin, Hindi +- No real-time generation +- Cannot perfectly replicate specific people or copyrighted characters + +## Troubleshooting + +### aspect_ratio Parameter Error + +**Error**: `Extra inputs are not permitted [type=extra_forbidden, input_value='1:1', input_type=str]` + +**Cause**: The `aspect_ratio` parameter must be nested inside an `image_config` object, not passed directly to `GenerateContentConfig`. + +**Incorrect Usage**: +```python +# ❌ This will fail +config = types.GenerateContentConfig( + response_modalities=['image'], + aspect_ratio='16:9' # Wrong - not a direct parameter +) +``` + +**Correct Usage**: +```python +# ✅ Correct implementation +config = types.GenerateContentConfig( + response_modalities=['Image'], # Note: Capital 'I' + image_config=types.ImageConfig( + aspect_ratio='16:9' + ) +) +``` + +### Response Modality Case Sensitivity + +The `response_modalities` parameter expects capital case values: +- ✅ Correct: `['Image']`, `['Text']`, `['Image', 'Text']` +- ❌ Wrong: `['image']`, `['text']` diff --git a/skills/ai-multimodal/references/video-analysis.md b/skills/ai-multimodal/references/video-analysis.md new file mode 100644 index 0000000..09870af --- /dev/null +++ b/skills/ai-multimodal/references/video-analysis.md @@ -0,0 +1,502 @@ +# Video Analysis Reference + +Comprehensive guide for video understanding, temporal analysis, and YouTube processing using Gemini API. + +## Core Capabilities + +- **Video Summarization**: Create concise summaries +- **Question Answering**: Answer specific questions about content +- **Transcription**: Audio transcription with visual descriptions +- **Timestamp References**: Query specific moments (MM:SS format) +- **Video Clipping**: Process specific segments +- **Scene Detection**: Identify scene changes and transitions +- **Multiple Videos**: Compare up to 10 videos (2.5+) +- **YouTube Support**: Analyze YouTube videos directly +- **Custom Frame Rate**: Adjust FPS sampling + +## Supported Formats + +- MP4, MPEG, MOV, AVI, FLV, MPG, WebM, WMV, 3GPP + +## Model Selection + +### Gemini 2.5 Series +- **gemini-2.5-pro**: Best quality, 1M-2M context +- **gemini-2.5-flash**: Balanced, 1M-2M context +- **gemini-2.5-flash-preview-09-2025**: Preview features, 1M context + +### Gemini 2.0 Series +- **gemini-2.0-flash**: Fast processing +- **gemini-2.0-flash-lite**: Lightweight option + +### Context Windows +- **2M token models**: ~2 hours (default) or ~6 hours (low-res) +- **1M token models**: ~1 hour (default) or ~3 hours (low-res) + +## Basic Video Analysis + +### Local Video + +```python +from google import genai +import os + +client = genai.Client(api_key=os.getenv('GEMINI_API_KEY')) + +# Upload video (File API for >20MB) +myfile = client.files.upload(file='video.mp4') + +# Wait for processing +import time +while myfile.state.name == 'PROCESSING': + time.sleep(1) + myfile = client.files.get(name=myfile.name) + +if myfile.state.name == 'FAILED': + raise ValueError('Video processing failed') + +# Analyze +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Summarize this video in 3 key points', myfile] +) +print(response.text) +``` + +### YouTube Video + +```python +from google.genai import types + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Summarize the main topics discussed', + types.Part.from_uri( + uri='https://www.youtube.com/watch?v=VIDEO_ID', + mime_type='video/mp4' + ) + ] +) +``` + +### Inline Video (<20MB) + +```python +with open('short-clip.mp4', 'rb') as f: + video_bytes = f.read() + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'What happens in this video?', + types.Part.from_bytes(data=video_bytes, mime_type='video/mp4') + ] +) +``` + +## Advanced Features + +### Video Clipping + +```python +# Analyze specific time range +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Summarize this segment', + types.Part.from_video_metadata( + file_uri=myfile.uri, + start_offset='40s', + end_offset='80s' + ) + ] +) +``` + +### Custom Frame Rate + +```python +# Lower FPS for static content (saves tokens) +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Analyze this presentation', + types.Part.from_video_metadata( + file_uri=myfile.uri, + fps=0.5 # Sample every 2 seconds + ) + ] +) + +# Higher FPS for fast-moving content +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Analyze rapid movements in this sports video', + types.Part.from_video_metadata( + file_uri=myfile.uri, + fps=5 # Sample 5 times per second + ) + ] +) +``` + +### Multiple Videos (2.5+) + +```python +video1 = client.files.upload(file='demo1.mp4') +video2 = client.files.upload(file='demo2.mp4') + +# Wait for processing +for video in [video1, video2]: + while video.state.name == 'PROCESSING': + time.sleep(1) + video = client.files.get(name=video.name) + +response = client.models.generate_content( + model='gemini-2.5-pro', + contents=[ + 'Compare these two product demos. Which explains features better?', + video1, + video2 + ] +) +``` + +## Temporal Understanding + +### Timestamp-Based Questions + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'What happens at 01:15 and how does it relate to 02:30?', + myfile + ] +) +``` + +### Timeline Creation + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Create a timeline with timestamps: + - Key events + - Scene changes + - Important moments + Format: MM:SS - Description + ''', + myfile + ] +) +``` + +### Scene Detection + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Identify all scene changes with timestamps and describe each scene', + myfile + ] +) +``` + +## Transcription + +### Basic Transcription + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Transcribe the audio from this video', + myfile + ] +) +``` + +### With Visual Descriptions + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Transcribe with visual context: + - Audio transcription + - Visual descriptions of important moments + - Timestamps for salient events + ''', + myfile + ] +) +``` + +### Speaker Identification + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Transcribe with speaker labels and timestamps', + myfile + ] +) +``` + +## Common Use Cases + +### 1. Video Summarization + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Summarize this video: + 1. Main topic and purpose + 2. Key points with timestamps + 3. Conclusion or call-to-action + ''', + myfile + ] +) +``` + +### 2. Educational Content + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Create educational materials: + 1. List key concepts taught + 2. Create 5 quiz questions with answers + 3. Provide timestamp for each concept + ''', + myfile + ] +) +``` + +### 3. Action Detection + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'List all actions performed in this tutorial with timestamps', + myfile + ] +) +``` + +### 4. Content Moderation + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Review video content: + 1. Identify any problematic content + 2. Note timestamps of concerns + 3. Provide content rating recommendation + ''', + myfile + ] +) +``` + +### 5. Interview Analysis + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Analyze interview: + 1. Questions asked (timestamps) + 2. Key responses + 3. Candidate body language and demeanor + 4. Overall assessment + ''', + myfile + ] +) +``` + +### 6. Sports Analysis + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Analyze sports video: + 1. Key plays with timestamps + 2. Player movements and positioning + 3. Game strategy observations + ''', + types.Part.from_video_metadata( + file_uri=myfile.uri, + fps=5 # Higher FPS for fast action + ) + ] +) +``` + +## YouTube Specific Features + +### Public Video Requirements + +- Video must be public (not private or unlisted) +- No age-restricted content +- Valid video ID required + +### Usage Example + +```python +# YouTube URL +youtube_uri = 'https://www.youtube.com/watch?v=dQw4w9WgXcQ' + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Create chapter markers with timestamps', + types.Part.from_uri(uri=youtube_uri, mime_type='video/mp4') + ] +) +``` + +### Rate Limits + +- **Free tier**: 8 hours of YouTube video per day +- **Paid tier**: No length-based limits +- Public videos only + +## Token Calculation + +Video tokens depend on resolution and FPS: + +**Default resolution** (~300 tokens/second): +- 1 minute = 18,000 tokens +- 10 minutes = 180,000 tokens +- 1 hour = 1,080,000 tokens + +**Low resolution** (~100 tokens/second): +- 1 minute = 6,000 tokens +- 10 minutes = 60,000 tokens +- 1 hour = 360,000 tokens + +**Context windows**: +- 2M tokens ≈ 2 hours (default) or 6 hours (low-res) +- 1M tokens ≈ 1 hour (default) or 3 hours (low-res) + +## Best Practices + +### File Management + +1. Use File API for videos >20MB (most videos) +2. Wait for ACTIVE state before analysis +3. Files auto-delete after 48 hours +4. Clean up manually: + ```python + client.files.delete(name=myfile.name) + ``` + +### Optimization Strategies + +**Reduce token usage**: +- Process specific segments using start/end offsets +- Use lower FPS for static content +- Use low-resolution mode for long videos +- Split very long videos into chunks + +**Improve accuracy**: +- Provide context in prompts +- Use higher FPS for fast-moving content +- Use Pro model for complex analysis +- Be specific about what to extract + +### Prompt Engineering + +**Effective prompts**: +- "Summarize key points with timestamps in MM:SS format" +- "Identify all scene changes and describe each scene" +- "Extract action items mentioned with timestamps" +- "Compare these two videos on: X, Y, Z criteria" + +**Structured output**: +```python +from pydantic import BaseModel +from typing import List + +class VideoEvent(BaseModel): + timestamp: str # MM:SS format + description: str + category: str + +class VideoAnalysis(BaseModel): + summary: str + events: List[VideoEvent] + duration: str + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Analyze this video', myfile], + config=genai.types.GenerateContentConfig( + response_mime_type='application/json', + response_schema=VideoAnalysis + ) +) +``` + +### Error Handling + +```python +import time + +def upload_and_process_video(file_path, max_wait=300): + """Upload video and wait for processing""" + myfile = client.files.upload(file=file_path) + + elapsed = 0 + while myfile.state.name == 'PROCESSING' and elapsed < max_wait: + time.sleep(5) + myfile = client.files.get(name=myfile.name) + elapsed += 5 + + if myfile.state.name == 'FAILED': + raise ValueError(f'Video processing failed: {myfile.state.name}') + + if myfile.state.name == 'PROCESSING': + raise TimeoutError(f'Processing timeout after {max_wait}s') + + return myfile +``` + +## Cost Optimization + +**Token costs** (Gemini 2.5 Flash at $1/1M): +- 1 minute video (default): 18,000 tokens = $0.018 +- 10 minute video: 180,000 tokens = $0.18 +- 1 hour video: 1,080,000 tokens = $1.08 + +**Strategies**: +- Use video clipping for specific segments +- Lower FPS for static content +- Use low-resolution mode for long videos +- Batch related queries on same video +- Use context caching for repeated queries + +## Limitations + +- Maximum 6 hours (low-res) or 2 hours (default) +- YouTube videos must be public +- No live streaming analysis +- Files expire after 48 hours +- Processing time varies by video length +- No real-time processing +- Limited to 10 videos per request (2.5+) diff --git a/skills/ai-multimodal/references/vision-understanding.md b/skills/ai-multimodal/references/vision-understanding.md new file mode 100644 index 0000000..5969d91 --- /dev/null +++ b/skills/ai-multimodal/references/vision-understanding.md @@ -0,0 +1,483 @@ +# Vision Understanding Reference + +Comprehensive guide for image analysis, object detection, and visual understanding using Gemini API. + +## Core Capabilities + +- **Captioning**: Generate descriptive text for images +- **Classification**: Categorize and identify content +- **Visual Q&A**: Answer questions about images +- **Object Detection**: Locate objects with bounding boxes (2.0+) +- **Segmentation**: Create pixel-level masks (2.5+) +- **Multi-image**: Compare up to 3,600 images +- **OCR**: Extract text from images +- **Document Understanding**: Process PDFs with vision + +## Supported Formats + +- **Images**: PNG, JPEG, WEBP, HEIC, HEIF +- **Documents**: PDF (up to 1,000 pages) +- **Size Limits**: + - Inline: 20MB max total request + - File API: 2GB per file + - Max images: 3,600 per request + +## Model Selection + +### Gemini 2.5 Series +- **gemini-2.5-pro**: Best quality, segmentation + detection +- **gemini-2.5-flash**: Fast, efficient, all features +- **gemini-2.5-flash-lite**: Lightweight, all features + +### Gemini 2.0 Series +- **gemini-2.0-flash**: Object detection support +- **gemini-2.0-flash-lite**: Lightweight detection + +### Feature Requirements +- **Segmentation**: Requires 2.5+ models +- **Object Detection**: Requires 2.0+ models +- **Multi-image**: All models (up to 3,600 images) + +## Basic Image Analysis + +### Image Captioning + +```python +from google import genai +import os + +client = genai.Client(api_key=os.getenv('GEMINI_API_KEY')) + +# Local file +with open('image.jpg', 'rb') as f: + img_bytes = f.read() + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Describe this image in detail', + genai.types.Part.from_bytes(data=img_bytes, mime_type='image/jpeg') + ] +) +print(response.text) +``` + +### Image Classification + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Classify this image. Provide category and confidence level.', + img_part + ] +) +``` + +### Visual Question Answering + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'How many people are in this image and what are they doing?', + img_part + ] +) +``` + +## Advanced Features + +### Object Detection (2.0+) + +```python +response = client.models.generate_content( + model='gemini-2.0-flash', + contents=[ + 'Detect all objects in this image and provide bounding boxes', + img_part + ] +) + +# Returns bounding box coordinates: [ymin, xmin, ymax, xmax] +# Normalized to [0, 1000] range +``` + +### Segmentation (2.5+) + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Create a segmentation mask for all people in this image', + img_part + ] +) + +# Returns pixel-level masks for requested objects +``` + +### Multi-Image Comparison + +```python +import PIL.Image + +img1 = PIL.Image.open('photo1.jpg') +img2 = PIL.Image.open('photo2.jpg') + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Compare these two images. What are the differences?', + img1, + img2 + ] +) +``` + +### OCR and Text Extraction + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Extract all visible text from this image', + img_part + ] +) +``` + +## Input Methods + +### Inline Data (<20MB) + +```python +from google.genai import types + +# From file +with open('image.jpg', 'rb') as f: + img_bytes = f.read() + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Analyze this image', + types.Part.from_bytes(data=img_bytes, mime_type='image/jpeg') + ] +) +``` + +### PIL Image + +```python +import PIL.Image + +img = PIL.Image.open('photo.jpg') + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['What is in this image?', img] +) +``` + +### File API (>20MB or Reuse) + +```python +# Upload once +myfile = client.files.upload(file='large-image.jpg') + +# Use multiple times +response1 = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Describe this image', myfile] +) + +response2 = client.models.generate_content( + model='gemini-2.5-flash', + contents=['What colors dominate this image?', myfile] +) +``` + +### URL (Public Images) + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Analyze this image', + types.Part.from_uri( + uri='https://example.com/image.jpg', + mime_type='image/jpeg' + ) + ] +) +``` + +## Token Calculation + +Images consume tokens based on size: + +**Small images** (≤384px both dimensions): 258 tokens + +**Large images**: Tiled into 768×768 chunks, 258 tokens each + +**Formula**: +``` +crop_unit = floor(min(width, height) / 1.5) +tiles = (width / crop_unit) × (height / crop_unit) +total_tokens = tiles × 258 +``` + +**Examples**: +- 256×256: 258 tokens (small) +- 512×512: 258 tokens (small) +- 960×540: 6 tiles = 1,548 tokens +- 1920×1080: 6 tiles = 1,548 tokens +- 3840×2160 (4K): 24 tiles = 6,192 tokens + +## Structured Output + +### JSON Schema Output + +```python +from pydantic import BaseModel +from typing import List + +class ObjectDetection(BaseModel): + object_name: str + confidence: float + bounding_box: List[int] # [ymin, xmin, ymax, xmax] + +class ImageAnalysis(BaseModel): + description: str + objects: List[ObjectDetection] + scene_type: str + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Analyze this image', img_part], + config=genai.types.GenerateContentConfig( + response_mime_type='application/json', + response_schema=ImageAnalysis + ) +) + +result = ImageAnalysis.model_validate_json(response.text) +``` + +## Multi-Image Analysis + +### Batch Processing + +```python +images = [ + PIL.Image.open(f'image{i}.jpg') + for i in range(10) +] + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=['Analyze these images and find common themes'] + images +) +``` + +### Image Comparison + +```python +before = PIL.Image.open('before.jpg') +after = PIL.Image.open('after.jpg') + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Compare before and after. List all visible changes.', + before, + after + ] +) +``` + +### Visual Search + +```python +reference = PIL.Image.open('target.jpg') +candidates = [PIL.Image.open(f'option{i}.jpg') for i in range(5)] + +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Find which candidate images contain objects similar to the reference', + reference + ] + candidates +) +``` + +## Best Practices + +### Image Quality + +1. **Resolution**: Use clear, non-blurry images +2. **Rotation**: Verify correct orientation +3. **Lighting**: Ensure good contrast and lighting +4. **Size optimization**: Balance quality vs token cost +5. **Format**: JPEG for photos, PNG for graphics + +### Prompt Engineering + +**Specific instructions**: +- "Identify all vehicles with their colors and positions" +- "Count people wearing blue shirts" +- "Extract text from the sign in the top-left corner" + +**Output format**: +- "Return results as JSON with fields: category, count, description" +- "Format as markdown table" +- "List findings as numbered items" + +**Few-shot examples**: +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Example: For an image of a cat on a sofa, respond: "Object: cat, Location: sofa"', + 'Now analyze this image:', + img_part + ] +) +``` + +### File Management + +1. Use File API for images >20MB +2. Use File API for repeated queries (saves tokens) +3. Files auto-delete after 48 hours +4. Clean up manually: + ```python + client.files.delete(name=myfile.name) + ``` + +### Cost Optimization + +**Token-efficient strategies**: +- Resize large images before upload +- Use File API for repeated queries +- Batch multiple images when related +- Use appropriate model (Flash vs Pro) + +**Token costs** (Gemini 2.5 Flash at $1/1M): +- Small image (258 tokens): $0.000258 +- HD image (1,548 tokens): $0.001548 +- 4K image (6,192 tokens): $0.006192 + +## Common Use Cases + +### 1. Product Analysis + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Analyze this product image: + 1. Identify the product + 2. List visible features + 3. Assess condition + 4. Estimate value range + ''', + img_part + ] +) +``` + +### 2. Screenshot Analysis + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Extract all text and UI elements from this screenshot', + img_part + ] +) +``` + +### 3. Medical Imaging (Informational Only) + +```python +response = client.models.generate_content( + model='gemini-2.5-pro', + contents=[ + 'Describe visible features in this medical image. Note: This is for informational purposes only.', + img_part + ] +) +``` + +### 4. Chart/Graph Reading + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + 'Extract data from this chart and format as JSON', + img_part + ] +) +``` + +### 5. Scene Understanding + +```python +response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + '''Analyze this scene: + 1. Location type + 2. Time of day + 3. Weather conditions + 4. Activities happening + 5. Mood/atmosphere + ''', + img_part + ] +) +``` + +## Error Handling + +```python +import time + +def analyze_image_with_retry(image_path, prompt, max_retries=3): + """Analyze image with exponential backoff retry""" + for attempt in range(max_retries): + try: + with open(image_path, 'rb') as f: + img_bytes = f.read() + + response = client.models.generate_content( + model='gemini-2.5-flash', + contents=[ + prompt, + genai.types.Part.from_bytes( + data=img_bytes, + mime_type='image/jpeg' + ) + ] + ) + return response.text + except Exception as e: + if attempt == max_retries - 1: + raise + wait_time = 2 ** attempt + print(f"Retry {attempt + 1} after {wait_time}s: {e}") + time.sleep(wait_time) +``` + +## Limitations + +- Maximum 3,600 images per request +- OCR accuracy varies with text quality +- Object detection requires 2.0+ models +- Segmentation requires 2.5+ models +- No video frame extraction (use video API) +- Regional restrictions on child images (EEA, CH, UK) diff --git a/skills/ai-multimodal/scripts/document_converter.py b/skills/ai-multimodal/scripts/document_converter.py new file mode 100644 index 0000000..8cc7134 --- /dev/null +++ b/skills/ai-multimodal/scripts/document_converter.py @@ -0,0 +1,395 @@ +#!/usr/bin/env python3 +""" +Convert documents to Markdown using Gemini API. + +Supports all document types: +- PDF documents (native vision processing) +- Images (JPEG, PNG, WEBP, HEIC) +- Office documents (DOCX, XLSX, PPTX) +- HTML, TXT, and other text formats + +Features: +- Converts to clean markdown format +- Preserves structure, tables, and formatting +- Extracts text from images and scanned documents +- Batch conversion support +- Saves to docs/assets/document-extraction.md by default +""" + +import argparse +import os +import sys +import time +from pathlib import Path +from typing import Optional, List, Dict, Any + +try: + from google import genai + from google.genai import types +except ImportError: + print("Error: google-genai package not installed") + print("Install with: pip install google-genai") + sys.exit(1) + +try: + from dotenv import load_dotenv +except ImportError: + load_dotenv = None + + +def find_api_key() -> Optional[str]: + """Find Gemini API key using correct priority order. + + Priority order (highest to lowest): + 1. process.env (runtime environment variables) + 2. .claude/skills/ai-multimodal/.env (skill-specific config) + 3. .claude/skills/.env (shared skills config) + 4. .claude/.env (Claude global config) + """ + # Priority 1: Already in process.env (highest) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + # Load .env files if dotenv available + if load_dotenv: + # Determine base paths + script_dir = Path(__file__).parent + skill_dir = script_dir.parent # .claude/skills/ai-multimodal + skills_dir = skill_dir.parent # .claude/skills + claude_dir = skills_dir.parent # .claude + + # Priority 2: Skill-specific .env + env_file = skill_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + # Priority 3: Shared skills .env + env_file = skills_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + # Priority 4: Claude global .env + env_file = claude_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + return None + + +def find_project_root() -> Path: + """Find project root directory.""" + script_dir = Path(__file__).parent + + # Look for .git or .claude directory + for parent in [script_dir] + list(script_dir.parents): + if (parent / '.git').exists() or (parent / '.claude').exists(): + return parent + + return script_dir + + +def get_mime_type(file_path: str) -> str: + """Determine MIME type from file extension.""" + ext = Path(file_path).suffix.lower() + + mime_types = { + # Documents + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.html': 'text/html', + '.htm': 'text/html', + '.md': 'text/markdown', + '.csv': 'text/csv', + # Images + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.webp': 'image/webp', + '.heic': 'image/heic', + '.heif': 'image/heif', + # Office (need to be uploaded as binary) + '.docx': 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', + '.xlsx': 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', + '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + } + + return mime_types.get(ext, 'application/octet-stream') + + +def upload_file(client: genai.Client, file_path: str, verbose: bool = False) -> Any: + """Upload file to Gemini File API.""" + if verbose: + print(f"Uploading {file_path}...") + + myfile = client.files.upload(file=file_path) + + # Wait for processing if needed + max_wait = 300 # 5 minutes + elapsed = 0 + while myfile.state.name == 'PROCESSING' and elapsed < max_wait: + time.sleep(2) + myfile = client.files.get(name=myfile.name) + elapsed += 2 + if verbose and elapsed % 10 == 0: + print(f" Processing... {elapsed}s") + + if myfile.state.name == 'FAILED': + raise ValueError(f"File processing failed: {file_path}") + + if myfile.state.name == 'PROCESSING': + raise TimeoutError(f"Processing timeout after {max_wait}s: {file_path}") + + if verbose: + print(f" Uploaded: {myfile.name}") + + return myfile + + +def convert_to_markdown( + client: genai.Client, + file_path: str, + model: str = 'gemini-2.5-flash', + custom_prompt: Optional[str] = None, + verbose: bool = False, + max_retries: int = 3 +) -> Dict[str, Any]: + """Convert a document to markdown using Gemini.""" + + for attempt in range(max_retries): + try: + file_path_obj = Path(file_path) + file_size = file_path_obj.stat().st_size + use_file_api = file_size > 20 * 1024 * 1024 # >20MB + + # Default prompt for markdown conversion + if custom_prompt: + prompt = custom_prompt + else: + prompt = """Convert this document to clean, well-formatted Markdown. + +Requirements: +- Preserve all content, structure, and formatting +- Convert tables to markdown table format +- Maintain heading hierarchy (# ## ### etc) +- Preserve lists, code blocks, and quotes +- Extract text from images if present +- Keep formatting consistent and readable + +Output only the markdown content without any preamble or explanation.""" + + # Upload or inline the file + if use_file_api: + myfile = upload_file(client, str(file_path), verbose) + content = [prompt, myfile] + else: + with open(file_path, 'rb') as f: + file_bytes = f.read() + + mime_type = get_mime_type(str(file_path)) + content = [ + prompt, + types.Part.from_bytes(data=file_bytes, mime_type=mime_type) + ] + + # Generate markdown + response = client.models.generate_content( + model=model, + contents=content + ) + + markdown_content = response.text if hasattr(response, 'text') else '' + + return { + 'file': str(file_path), + 'status': 'success', + 'markdown': markdown_content + } + + except Exception as e: + if attempt == max_retries - 1: + return { + 'file': str(file_path), + 'status': 'error', + 'error': str(e), + 'markdown': None + } + + wait_time = 2 ** attempt + if verbose: + print(f" Retry {attempt + 1} after {wait_time}s: {e}") + time.sleep(wait_time) + + +def batch_convert( + files: List[str], + output_file: Optional[str] = None, + auto_name: bool = False, + model: str = 'gemini-2.5-flash', + custom_prompt: Optional[str] = None, + verbose: bool = False +) -> List[Dict[str, Any]]: + """Batch convert multiple files to markdown.""" + + api_key = find_api_key() + if not api_key: + print("Error: GEMINI_API_KEY not found") + print("Set via: export GEMINI_API_KEY='your-key'") + print("Or create .env file with: GEMINI_API_KEY=your-key") + sys.exit(1) + + client = genai.Client(api_key=api_key) + results = [] + + # Determine output path + if not output_file: + project_root = find_project_root() + output_dir = project_root / 'docs' / 'assets' + + if auto_name and len(files) == 1: + # Auto-generate meaningful filename from input + input_path = Path(files[0]) + base_name = input_path.stem + output_file = str(output_dir / f"{base_name}-extraction.md") + else: + output_file = str(output_dir / 'document-extraction.md') + + output_path = Path(output_file) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Process each file + for i, file_path in enumerate(files, 1): + if verbose: + print(f"\n[{i}/{len(files)}] Converting: {file_path}") + + result = convert_to_markdown( + client=client, + file_path=file_path, + model=model, + custom_prompt=custom_prompt, + verbose=verbose + ) + + results.append(result) + + if verbose: + status = result.get('status', 'unknown') + print(f" Status: {status}") + + # Save combined markdown + with open(output_path, 'w', encoding='utf-8') as f: + f.write("# Document Extraction Results\n\n") + f.write(f"Converted {len(files)} document(s) to markdown.\n\n") + f.write("---\n\n") + + for result in results: + f.write(f"## {Path(result['file']).name}\n\n") + + if result['status'] == 'success' and result.get('markdown'): + f.write(result['markdown']) + f.write("\n\n") + elif result['status'] == 'success': + f.write("**Note**: Conversion succeeded but no content was returned.\n\n") + else: + f.write(f"**Error**: {result.get('error', 'Unknown error')}\n\n") + + f.write("---\n\n") + + if verbose or True: # Always show output location + print(f"\n{'='*50}") + print(f"Converted: {len(results)} file(s)") + print(f"Success: {sum(1 for r in results if r['status'] == 'success')}") + print(f"Failed: {sum(1 for r in results if r['status'] == 'error')}") + print(f"Output saved to: {output_path}") + + return results + + +def main(): + parser = argparse.ArgumentParser( + description='Convert documents to Markdown using Gemini API', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Convert single PDF to markdown (default name) + %(prog)s --input document.pdf + + # Auto-generate meaningful filename + %(prog)s --input testpdf.pdf --auto-name + # Output: docs/assets/testpdf-extraction.md + + # Convert multiple files + %(prog)s --input doc1.pdf doc2.docx image.png + + # Specify custom output location + %(prog)s --input document.pdf --output ./output.md + + # Use custom prompt + %(prog)s --input document.pdf --prompt "Extract only the tables as markdown" + + # Batch convert directory + %(prog)s --input ./documents/*.pdf --verbose + +Supported formats: + - PDF documents (up to 1,000 pages) + - Images (JPEG, PNG, WEBP, HEIC) + - Office documents (DOCX, XLSX, PPTX) + - Text formats (TXT, HTML, Markdown, CSV) + +Default output: /docs/assets/document-extraction.md + """ + ) + + parser.add_argument('--input', '-i', nargs='+', required=True, + help='Input file(s) to convert') + parser.add_argument('--output', '-o', + help='Output markdown file (default: docs/assets/document-extraction.md)') + parser.add_argument('--auto-name', '-a', action='store_true', + help='Auto-generate meaningful output filename from input (e.g., document.pdf -> document-extraction.md)') + parser.add_argument('--model', default='gemini-2.5-flash', + help='Gemini model to use (default: gemini-2.5-flash)') + parser.add_argument('--prompt', '-p', + help='Custom prompt for conversion') + parser.add_argument('--verbose', '-v', action='store_true', + help='Verbose output') + + args = parser.parse_args() + + # Validate input files + files = [] + for file_pattern in args.input: + file_path = Path(file_pattern) + if file_path.exists() and file_path.is_file(): + files.append(str(file_path)) + else: + # Try glob pattern + import glob + matched = glob.glob(file_pattern) + files.extend([f for f in matched if Path(f).is_file()]) + + if not files: + print("Error: No valid input files found") + sys.exit(1) + + # Convert files + batch_convert( + files=files, + output_file=args.output, + auto_name=args.auto_name, + model=args.model, + custom_prompt=args.prompt, + verbose=args.verbose + ) + + +if __name__ == '__main__': + main() diff --git a/skills/ai-multimodal/scripts/gemini_batch_process.py b/skills/ai-multimodal/scripts/gemini_batch_process.py new file mode 100644 index 0000000..f65a13f --- /dev/null +++ b/skills/ai-multimodal/scripts/gemini_batch_process.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +""" +Batch process multiple media files using Gemini API. + +Supports all Gemini modalities: +- Audio: Transcription, analysis, summarization +- Image: Captioning, detection, OCR, analysis +- Video: Summarization, Q&A, scene detection +- Document: PDF extraction, structured output +- Generation: Image creation from text prompts +""" + +import argparse +import json +import os +import sys +import time +from pathlib import Path +from typing import List, Dict, Any, Optional +import csv +import shutil + +try: + from google import genai + from google.genai import types +except ImportError: + print("Error: google-genai package not installed") + print("Install with: pip install google-genai") + sys.exit(1) + +try: + from dotenv import load_dotenv +except ImportError: + load_dotenv = None + + +def find_api_key() -> Optional[str]: + """Find Gemini API key using correct priority order. + + Priority order (highest to lowest): + 1. process.env (runtime environment variables) + 2. .claude/skills/ai-multimodal/.env (skill-specific config) + 3. .claude/skills/.env (shared skills config) + 4. .claude/.env (Claude global config) + """ + # Priority 1: Already in process.env (highest) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + # Load .env files if dotenv available + if load_dotenv: + # Determine base paths + script_dir = Path(__file__).parent + skill_dir = script_dir.parent # .claude/skills/ai-multimodal + skills_dir = skill_dir.parent # .claude/skills + claude_dir = skills_dir.parent # .claude + + # Priority 2: Skill-specific .env + env_file = skill_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + # Priority 3: Shared skills .env + env_file = skills_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + # Priority 4: Claude global .env + env_file = claude_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + api_key = os.getenv('GEMINI_API_KEY') + if api_key: + return api_key + + return None + + +def get_mime_type(file_path: str) -> str: + """Determine MIME type from file extension.""" + ext = Path(file_path).suffix.lower() + + mime_types = { + # Audio + '.mp3': 'audio/mp3', + '.wav': 'audio/wav', + '.aac': 'audio/aac', + '.flac': 'audio/flac', + '.ogg': 'audio/ogg', + '.aiff': 'audio/aiff', + # Image + '.jpg': 'image/jpeg', + '.jpeg': 'image/jpeg', + '.png': 'image/png', + '.webp': 'image/webp', + '.heic': 'image/heic', + '.heif': 'image/heif', + # Video + '.mp4': 'video/mp4', + '.mpeg': 'video/mpeg', + '.mov': 'video/quicktime', + '.avi': 'video/x-msvideo', + '.flv': 'video/x-flv', + '.mpg': 'video/mpeg', + '.webm': 'video/webm', + '.wmv': 'video/x-ms-wmv', + '.3gpp': 'video/3gpp', + # Document + '.pdf': 'application/pdf', + '.txt': 'text/plain', + '.html': 'text/html', + '.md': 'text/markdown', + } + + return mime_types.get(ext, 'application/octet-stream') + + +def upload_file(client: genai.Client, file_path: str, verbose: bool = False) -> Any: + """Upload file to Gemini File API.""" + if verbose: + print(f"Uploading {file_path}...") + + myfile = client.files.upload(file=file_path) + + # Wait for processing (video/audio files need processing) + mime_type = get_mime_type(file_path) + if mime_type.startswith('video/') or mime_type.startswith('audio/'): + max_wait = 300 # 5 minutes + elapsed = 0 + while myfile.state.name == 'PROCESSING' and elapsed < max_wait: + time.sleep(2) + myfile = client.files.get(name=myfile.name) + elapsed += 2 + if verbose and elapsed % 10 == 0: + print(f" Processing... {elapsed}s") + + if myfile.state.name == 'FAILED': + raise ValueError(f"File processing failed: {file_path}") + + if myfile.state.name == 'PROCESSING': + raise TimeoutError(f"Processing timeout after {max_wait}s: {file_path}") + + if verbose: + print(f" Uploaded: {myfile.name}") + + return myfile + + +def process_file( + client: genai.Client, + file_path: Optional[str], + prompt: str, + model: str, + task: str, + format_output: str, + aspect_ratio: Optional[str] = None, + verbose: bool = False, + max_retries: int = 3 +) -> Dict[str, Any]: + """Process a single file with retry logic.""" + + for attempt in range(max_retries): + try: + # For generation tasks without input files + if task == 'generate' and not file_path: + content = [prompt] + else: + # Process input file + file_path = Path(file_path) + # Determine if we need File API + file_size = file_path.stat().st_size + use_file_api = file_size > 20 * 1024 * 1024 # >20MB + + if use_file_api: + # Upload to File API + myfile = upload_file(client, str(file_path), verbose) + content = [prompt, myfile] + else: + # Inline data + with open(file_path, 'rb') as f: + file_bytes = f.read() + + mime_type = get_mime_type(str(file_path)) + content = [ + prompt, + types.Part.from_bytes(data=file_bytes, mime_type=mime_type) + ] + + # Configure request + config_args = {} + if task == 'generate': + config_args['response_modalities'] = ['Image'] # Capital I per API spec + if aspect_ratio: + # Nest aspect_ratio in image_config per API spec + config_args['image_config'] = types.ImageConfig( + aspect_ratio=aspect_ratio + ) + + if format_output == 'json': + config_args['response_mime_type'] = 'application/json' + + config = types.GenerateContentConfig(**config_args) if config_args else None + + # Generate content + response = client.models.generate_content( + model=model, + contents=content, + config=config + ) + + # Extract response + result = { + 'file': str(file_path) if file_path else 'generated', + 'status': 'success', + 'response': response.text if hasattr(response, 'text') else None + } + + # Handle image output + if task == 'generate' and hasattr(response, 'candidates'): + for i, part in enumerate(response.candidates[0].content.parts): + if part.inline_data: + # Determine output directory - use project root docs/assets + if file_path: + output_dir = Path(file_path).parent + base_name = Path(file_path).stem + else: + # Find project root (look for .git or .claude directory) + script_dir = Path(__file__).parent + project_root = script_dir + for parent in [script_dir] + list(script_dir.parents): + if (parent / '.git').exists() or (parent / '.claude').exists(): + project_root = parent + break + + output_dir = project_root / 'docs' / 'assets' + output_dir.mkdir(parents=True, exist_ok=True) + base_name = "generated" + + output_file = output_dir / f"{base_name}_generated_{i}.png" + with open(output_file, 'wb') as f: + f.write(part.inline_data.data) + result['generated_image'] = str(output_file) + if verbose: + print(f" Saved image to: {output_file}") + + return result + + except Exception as e: + if attempt == max_retries - 1: + return { + 'file': str(file_path) if file_path else 'generated', + 'status': 'error', + 'error': str(e) + } + + wait_time = 2 ** attempt + if verbose: + print(f" Retry {attempt + 1} after {wait_time}s: {e}") + time.sleep(wait_time) + + +def batch_process( + files: List[str], + prompt: str, + model: str, + task: str, + format_output: str, + aspect_ratio: Optional[str] = None, + output_file: Optional[str] = None, + verbose: bool = False, + dry_run: bool = False +) -> List[Dict[str, Any]]: + """Batch process multiple files.""" + api_key = find_api_key() + if not api_key: + print("Error: GEMINI_API_KEY not found") + print("Set via: export GEMINI_API_KEY='your-key'") + print("Or create .env file with: GEMINI_API_KEY=your-key") + sys.exit(1) + + if dry_run: + print("DRY RUN MODE - No API calls will be made") + print(f"Files to process: {len(files)}") + print(f"Model: {model}") + print(f"Task: {task}") + print(f"Prompt: {prompt}") + return [] + + client = genai.Client(api_key=api_key) + results = [] + + # For generation tasks without input files, process once + if task == 'generate' and not files: + if verbose: + print(f"\nGenerating image from prompt...") + + result = process_file( + client=client, + file_path=None, + prompt=prompt, + model=model, + task=task, + format_output=format_output, + aspect_ratio=aspect_ratio, + verbose=verbose + ) + + results.append(result) + + if verbose: + status = result.get('status', 'unknown') + print(f" Status: {status}") + else: + # Process input files + for i, file_path in enumerate(files, 1): + if verbose: + print(f"\n[{i}/{len(files)}] Processing: {file_path}") + + result = process_file( + client=client, + file_path=file_path, + prompt=prompt, + model=model, + task=task, + format_output=format_output, + aspect_ratio=aspect_ratio, + verbose=verbose + ) + + results.append(result) + + if verbose: + status = result.get('status', 'unknown') + print(f" Status: {status}") + + # Save results + if output_file: + save_results(results, output_file, format_output) + + return results + + +def save_results(results: List[Dict[str, Any]], output_file: str, format_output: str): + """Save results to file.""" + output_path = Path(output_file) + + # Special handling for image generation - if output has image extension, copy the generated image + image_extensions = {'.png', '.jpg', '.jpeg', '.webp', '.gif', '.bmp'} + if output_path.suffix.lower() in image_extensions and len(results) == 1: + generated_image = results[0].get('generated_image') + if generated_image: + # Copy the generated image to the specified output location + shutil.copy2(generated_image, output_path) + return + else: + # Don't write text reports to image files - save error as .txt instead + output_path = output_path.with_suffix('.error.txt') + print(f"Warning: Generation failed, saving error report to: {output_path}") + + if format_output == 'json': + with open(output_path, 'w') as f: + json.dump(results, f, indent=2) + elif format_output == 'csv': + with open(output_path, 'w', newline='') as f: + fieldnames = ['file', 'status', 'response', 'error'] + writer = csv.DictWriter(f, fieldnames=fieldnames) + writer.writeheader() + for result in results: + writer.writerow({ + 'file': result.get('file', ''), + 'status': result.get('status', ''), + 'response': result.get('response', ''), + 'error': result.get('error', '') + }) + else: # markdown + with open(output_path, 'w') as f: + f.write("# Batch Processing Results\n\n") + for i, result in enumerate(results, 1): + f.write(f"## {i}. {result.get('file', 'Unknown')}\n\n") + f.write(f"**Status**: {result.get('status', 'unknown')}\n\n") + if result.get('response'): + f.write(f"**Response**:\n\n{result['response']}\n\n") + if result.get('error'): + f.write(f"**Error**: {result['error']}\n\n") + + +def main(): + parser = argparse.ArgumentParser( + description='Batch process media files with Gemini API', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Transcribe multiple audio files + %(prog)s --files *.mp3 --task transcribe --model gemini-2.5-flash + + # Analyze images + %(prog)s --files *.jpg --task analyze --prompt "Describe this image" \\ + --model gemini-2.5-flash + + # Process PDFs to JSON + %(prog)s --files *.pdf --task extract --prompt "Extract data as JSON" \\ + --format json --output results.json + + # Generate images + %(prog)s --task generate --prompt "A mountain landscape" \\ + --model gemini-2.5-flash-image --aspect-ratio 16:9 + """ + ) + + parser.add_argument('--files', nargs='*', help='Input files to process') + parser.add_argument('--task', required=True, + choices=['transcribe', 'analyze', 'extract', 'generate'], + help='Task to perform') + parser.add_argument('--prompt', help='Prompt for analysis/generation') + parser.add_argument('--model', default='gemini-2.5-flash', + help='Gemini model to use (default: gemini-2.5-flash)') + parser.add_argument('--format', dest='format_output', default='text', + choices=['text', 'json', 'csv', 'markdown'], + help='Output format (default: text)') + parser.add_argument('--aspect-ratio', choices=['1:1', '16:9', '9:16', '4:3', '3:4'], + help='Aspect ratio for image generation') + parser.add_argument('--output', help='Output file for results') + parser.add_argument('--verbose', '-v', action='store_true', + help='Verbose output') + parser.add_argument('--dry-run', action='store_true', + help='Show what would be done without making API calls') + + args = parser.parse_args() + + # Validate arguments + if args.task != 'generate' and not args.files: + parser.error("--files required for non-generation tasks") + + if args.task == 'generate' and not args.prompt: + parser.error("--prompt required for generation task") + + if args.task != 'generate' and not args.prompt: + # Set default prompts + if args.task == 'transcribe': + args.prompt = 'Generate a transcript with timestamps' + elif args.task == 'analyze': + args.prompt = 'Analyze this content' + elif args.task == 'extract': + args.prompt = 'Extract key information' + + # Process files + files = args.files or [] + results = batch_process( + files=files, + prompt=args.prompt, + model=args.model, + task=args.task, + format_output=args.format_output, + aspect_ratio=args.aspect_ratio, + output_file=args.output, + verbose=args.verbose, + dry_run=args.dry_run + ) + + # Print summary + if not args.dry_run and results: + success = sum(1 for r in results if r.get('status') == 'success') + failed = len(results) - success + print(f"\n{'='*50}") + print(f"Processed: {len(results)} files") + print(f"Success: {success}") + print(f"Failed: {failed}") + if args.output: + print(f"Results saved to: {args.output}") + + +if __name__ == '__main__': + main() diff --git a/skills/ai-multimodal/scripts/media_optimizer.py b/skills/ai-multimodal/scripts/media_optimizer.py new file mode 100644 index 0000000..06254b6 --- /dev/null +++ b/skills/ai-multimodal/scripts/media_optimizer.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python3 +""" +Optimize media files for Gemini API processing. + +Features: +- Compress videos/audio for size limits +- Resize images appropriately +- Split long videos into chunks +- Format conversion +- Quality vs size optimization +- Validation before upload +""" + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path +from typing import Optional, Dict, Any, List + +try: + from dotenv import load_dotenv +except ImportError: + load_dotenv = None + + +def load_env_files(): + """Load .env files in correct priority order. + + Priority order (highest to lowest): + 1. process.env (runtime environment variables) + 2. .claude/skills/ai-multimodal/.env (skill-specific config) + 3. .claude/skills/.env (shared skills config) + 4. .claude/.env (Claude global config) + """ + if not load_dotenv: + return + + # Determine base paths + script_dir = Path(__file__).parent + skill_dir = script_dir.parent # .claude/skills/ai-multimodal + skills_dir = skill_dir.parent # .claude/skills + claude_dir = skills_dir.parent # .claude + + # Priority 2: Skill-specific .env + env_file = skill_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + + # Priority 3: Shared skills .env + env_file = skills_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + + # Priority 4: Claude global .env + env_file = claude_dir / '.env' + if env_file.exists(): + load_dotenv(env_file) + + +# Load environment variables at module level +load_env_files() + + +def check_ffmpeg() -> bool: + """Check if ffmpeg is installed.""" + try: + subprocess.run(['ffmpeg', '-version'], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + check=True) + return True + except (subprocess.CalledProcessError, FileNotFoundError, Exception): + return False + + +def get_media_info(file_path: str) -> Dict[str, Any]: + """Get media file information using ffprobe.""" + if not check_ffmpeg(): + return {} + + try: + cmd = [ + 'ffprobe', + '-v', 'quiet', + '-print_format', 'json', + '-show_format', + '-show_streams', + file_path + ] + + result = subprocess.run(cmd, capture_output=True, text=True, check=True) + data = json.loads(result.stdout) + + info = { + 'size': int(data['format'].get('size', 0)), + 'duration': float(data['format'].get('duration', 0)), + 'bit_rate': int(data['format'].get('bit_rate', 0)), + } + + # Get video/audio specific info + for stream in data.get('streams', []): + if stream['codec_type'] == 'video': + info['width'] = stream.get('width', 0) + info['height'] = stream.get('height', 0) + info['fps'] = eval(stream.get('r_frame_rate', '0/1')) + elif stream['codec_type'] == 'audio': + info['sample_rate'] = int(stream.get('sample_rate', 0)) + info['channels'] = stream.get('channels', 0) + + return info + + except (subprocess.CalledProcessError, json.JSONDecodeError, Exception): + return {} + + +def optimize_video( + input_path: str, + output_path: str, + target_size_mb: Optional[int] = None, + max_duration: Optional[int] = None, + quality: int = 23, + resolution: Optional[str] = None, + verbose: bool = False +) -> bool: + """Optimize video file for Gemini API.""" + if not check_ffmpeg(): + print("Error: ffmpeg not installed") + print("Install: apt-get install ffmpeg (Linux) or brew install ffmpeg (Mac)") + return False + + info = get_media_info(input_path) + if not info: + print(f"Error: Could not read media info from {input_path}") + return False + + if verbose: + print(f"Input: {Path(input_path).name}") + print(f" Size: {info['size'] / (1024*1024):.2f} MB") + print(f" Duration: {info['duration']:.2f}s") + if 'width' in info: + print(f" Resolution: {info['width']}x{info['height']}") + print(f" Bit rate: {info['bit_rate'] / 1000:.0f} kbps") + + # Build ffmpeg command + cmd = ['ffmpeg', '-i', input_path, '-y'] + + # Video codec + cmd.extend(['-c:v', 'libx264', '-crf', str(quality)]) + + # Resolution + if resolution: + cmd.extend(['-vf', f'scale={resolution}']) + elif 'width' in info and info['width'] > 1920: + cmd.extend(['-vf', 'scale=1920:-2']) # Max 1080p + + # Audio codec + cmd.extend(['-c:a', 'aac', '-b:a', '128k', '-ac', '2']) + + # Duration limit + if max_duration and info['duration'] > max_duration: + cmd.extend(['-t', str(max_duration)]) + + # Target size (rough estimate using bitrate) + if target_size_mb: + target_bits = target_size_mb * 8 * 1024 * 1024 + duration = min(info['duration'], max_duration) if max_duration else info['duration'] + target_bitrate = int(target_bits / duration) + # Reserve some for audio (128kbps) + video_bitrate = max(target_bitrate - 128000, 500000) + cmd.extend(['-b:v', str(video_bitrate)]) + + cmd.append(output_path) + + if verbose: + print(f"\nOptimizing...") + print(f" Command: {' '.join(cmd)}") + + try: + subprocess.run(cmd, check=True, capture_output=not verbose) + + # Check output + output_info = get_media_info(output_path) + if output_info and verbose: + print(f"\nOutput: {Path(output_path).name}") + print(f" Size: {output_info['size'] / (1024*1024):.2f} MB") + print(f" Duration: {output_info['duration']:.2f}s") + if 'width' in output_info: + print(f" Resolution: {output_info['width']}x{output_info['height']}") + compression = (1 - output_info['size'] / info['size']) * 100 + print(f" Compression: {compression:.1f}%") + + return True + + except subprocess.CalledProcessError as e: + print(f"Error optimizing video: {e}") + return False + + +def optimize_audio( + input_path: str, + output_path: str, + target_size_mb: Optional[int] = None, + bitrate: str = '64k', + sample_rate: int = 16000, + verbose: bool = False +) -> bool: + """Optimize audio file for Gemini API.""" + if not check_ffmpeg(): + print("Error: ffmpeg not installed") + return False + + info = get_media_info(input_path) + if not info: + print(f"Error: Could not read media info from {input_path}") + return False + + if verbose: + print(f"Input: {Path(input_path).name}") + print(f" Size: {info['size'] / (1024*1024):.2f} MB") + print(f" Duration: {info['duration']:.2f}s") + + # Build command + cmd = [ + 'ffmpeg', '-i', input_path, '-y', + '-c:a', 'aac', + '-b:a', bitrate, + '-ar', str(sample_rate), + '-ac', '1', # Mono (Gemini uses mono anyway) + output_path + ] + + if verbose: + print(f"\nOptimizing...") + + try: + subprocess.run(cmd, check=True, capture_output=not verbose) + + output_info = get_media_info(output_path) + if output_info and verbose: + print(f"\nOutput: {Path(output_path).name}") + print(f" Size: {output_info['size'] / (1024*1024):.2f} MB") + compression = (1 - output_info['size'] / info['size']) * 100 + print(f" Compression: {compression:.1f}%") + + return True + + except subprocess.CalledProcessError as e: + print(f"Error optimizing audio: {e}") + return False + + +def optimize_image( + input_path: str, + output_path: str, + max_width: int = 1920, + quality: int = 85, + verbose: bool = False +) -> bool: + """Optimize image file for Gemini API.""" + try: + from PIL import Image + except ImportError: + print("Error: Pillow not installed") + print("Install with: pip install pillow") + return False + + try: + img = Image.open(input_path) + + if verbose: + print(f"Input: {Path(input_path).name}") + print(f" Size: {Path(input_path).stat().st_size / 1024:.2f} KB") + print(f" Resolution: {img.width}x{img.height}") + + # Resize if needed + if img.width > max_width: + ratio = max_width / img.width + new_height = int(img.height * ratio) + img = img.resize((max_width, new_height), Image.Resampling.LANCZOS) + if verbose: + print(f" Resized to: {img.width}x{img.height}") + + # Convert RGBA to RGB if saving as JPEG + if output_path.lower().endswith('.jpg') or output_path.lower().endswith('.jpeg'): + if img.mode == 'RGBA': + rgb_img = Image.new('RGB', img.size, (255, 255, 255)) + rgb_img.paste(img, mask=img.split()[3]) + img = rgb_img + + # Save + img.save(output_path, quality=quality, optimize=True) + + if verbose: + print(f"\nOutput: {Path(output_path).name}") + print(f" Size: {Path(output_path).stat().st_size / 1024:.2f} KB") + compression = (1 - Path(output_path).stat().st_size / Path(input_path).stat().st_size) * 100 + print(f" Compression: {compression:.1f}%") + + return True + + except Exception as e: + print(f"Error optimizing image: {e}") + return False + + +def split_video( + input_path: str, + output_dir: str, + chunk_duration: int = 3600, + verbose: bool = False +) -> List[str]: + """Split long video into chunks.""" + if not check_ffmpeg(): + print("Error: ffmpeg not installed") + return [] + + info = get_media_info(input_path) + if not info: + return [] + + total_duration = info['duration'] + num_chunks = int(total_duration / chunk_duration) + 1 + + if num_chunks == 1: + if verbose: + print("Video is short enough, no splitting needed") + return [input_path] + + Path(output_dir).mkdir(parents=True, exist_ok=True) + output_files = [] + + for i in range(num_chunks): + start_time = i * chunk_duration + output_file = Path(output_dir) / f"{Path(input_path).stem}_chunk_{i+1}.mp4" + + cmd = [ + 'ffmpeg', '-i', input_path, '-y', + '-ss', str(start_time), + '-t', str(chunk_duration), + '-c', 'copy', + str(output_file) + ] + + if verbose: + print(f"Creating chunk {i+1}/{num_chunks}...") + + try: + subprocess.run(cmd, check=True, capture_output=not verbose) + output_files.append(str(output_file)) + except subprocess.CalledProcessError as e: + print(f"Error creating chunk {i+1}: {e}") + + return output_files + + +def main(): + parser = argparse.ArgumentParser( + description='Optimize media files for Gemini API', + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Optimize video to 100MB + %(prog)s --input video.mp4 --output optimized.mp4 --target-size 100 + + # Optimize audio + %(prog)s --input audio.mp3 --output optimized.m4a --bitrate 64k + + # Resize image + %(prog)s --input image.jpg --output resized.jpg --max-width 1920 + + # Split long video + %(prog)s --input long-video.mp4 --split --chunk-duration 3600 --output-dir ./chunks + + # Batch optimize directory + %(prog)s --input-dir ./videos --output-dir ./optimized --quality 85 + """ + ) + + parser.add_argument('--input', help='Input file') + parser.add_argument('--output', help='Output file') + parser.add_argument('--input-dir', help='Input directory for batch processing') + parser.add_argument('--output-dir', help='Output directory for batch processing') + parser.add_argument('--target-size', type=int, help='Target size in MB') + parser.add_argument('--quality', type=int, default=85, + help='Quality (video: 0-51 CRF, image: 1-100) (default: 85)') + parser.add_argument('--max-width', type=int, default=1920, + help='Max image width (default: 1920)') + parser.add_argument('--bitrate', default='64k', + help='Audio bitrate (default: 64k)') + parser.add_argument('--resolution', help='Video resolution (e.g., 1920x1080)') + parser.add_argument('--split', action='store_true', help='Split long video into chunks') + parser.add_argument('--chunk-duration', type=int, default=3600, + help='Chunk duration in seconds (default: 3600 = 1 hour)') + parser.add_argument('--verbose', '-v', action='store_true', help='Verbose output') + + args = parser.parse_args() + + # Validate arguments + if not args.input and not args.input_dir: + parser.error("Either --input or --input-dir required") + + # Single file processing + if args.input: + input_path = Path(args.input) + if not input_path.exists(): + print(f"Error: Input file not found: {input_path}") + sys.exit(1) + + if args.split: + output_dir = args.output_dir or './chunks' + chunks = split_video(str(input_path), output_dir, args.chunk_duration, args.verbose) + print(f"\nCreated {len(chunks)} chunks in {output_dir}") + sys.exit(0) + + if not args.output: + parser.error("--output required for single file processing") + + output_path = Path(args.output) + output_path.parent.mkdir(parents=True, exist_ok=True) + + # Determine file type + ext = input_path.suffix.lower() + + if ext in ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.flv']: + success = optimize_video( + str(input_path), + str(output_path), + target_size_mb=args.target_size, + quality=args.quality, + resolution=args.resolution, + verbose=args.verbose + ) + elif ext in ['.mp3', '.wav', '.m4a', '.flac', '.aac']: + success = optimize_audio( + str(input_path), + str(output_path), + target_size_mb=args.target_size, + bitrate=args.bitrate, + verbose=args.verbose + ) + elif ext in ['.jpg', '.jpeg', '.png', '.webp']: + success = optimize_image( + str(input_path), + str(output_path), + max_width=args.max_width, + quality=args.quality, + verbose=args.verbose + ) + else: + print(f"Error: Unsupported file type: {ext}") + sys.exit(1) + + sys.exit(0 if success else 1) + + # Batch processing + if args.input_dir: + if not args.output_dir: + parser.error("--output-dir required for batch processing") + + input_dir = Path(args.input_dir) + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Find all media files + patterns = ['*.mp4', '*.mov', '*.avi', '*.mkv', '*.webm', + '*.mp3', '*.wav', '*.m4a', '*.flac', + '*.jpg', '*.jpeg', '*.png', '*.webp'] + + files = [] + for pattern in patterns: + files.extend(input_dir.glob(pattern)) + + if not files: + print(f"No media files found in {input_dir}") + sys.exit(1) + + print(f"Found {len(files)} files to process") + + success_count = 0 + for input_file in files: + output_file = output_dir / input_file.name + + ext = input_file.suffix.lower() + success = False + + if ext in ['.mp4', '.mov', '.avi', '.mkv', '.webm', '.flv']: + success = optimize_video(str(input_file), str(output_file), + quality=args.quality, verbose=args.verbose) + elif ext in ['.mp3', '.wav', '.m4a', '.flac', '.aac']: + success = optimize_audio(str(input_file), str(output_file), + bitrate=args.bitrate, verbose=args.verbose) + elif ext in ['.jpg', '.jpeg', '.png', '.webp']: + success = optimize_image(str(input_file), str(output_file), + max_width=args.max_width, quality=args.quality, + verbose=args.verbose) + + if success: + success_count += 1 + + print(f"\nProcessed: {success_count}/{len(files)} files") + + +if __name__ == '__main__': + main() diff --git a/skills/ai-multimodal/scripts/requirements.txt b/skills/ai-multimodal/scripts/requirements.txt new file mode 100644 index 0000000..7a67dac --- /dev/null +++ b/skills/ai-multimodal/scripts/requirements.txt @@ -0,0 +1,26 @@ +# AI Multimodal Skill Dependencies +# Python 3.10+ required + +# Google Gemini API +google-genai>=0.1.0 + +# PDF processing +pypdf>=4.0.0 + +# Document conversion +python-docx>=1.0.0 +docx2pdf>=0.1.8 # Windows only, optional on Linux/macOS + +# Markdown processing +markdown>=3.5.0 + +# Image processing +Pillow>=10.0.0 + +# Environment variable management +python-dotenv>=1.0.0 + +# Testing dependencies (dev) +pytest>=8.0.0 +pytest-cov>=4.1.0 +pytest-mock>=3.12.0 diff --git a/skills/ai-multimodal/scripts/tests/requirements.txt b/skills/ai-multimodal/scripts/tests/requirements.txt new file mode 100644 index 0000000..bc19f96 --- /dev/null +++ b/skills/ai-multimodal/scripts/tests/requirements.txt @@ -0,0 +1,20 @@ +# Core dependencies +google-genai>=0.2.0 +python-dotenv>=1.0.0 + +# Image processing +pillow>=10.0.0 + +# PDF processing +pypdf>=3.0.0 + +# Document conversion +markdown>=3.5 + +# Testing +pytest>=7.4.0 +pytest-cov>=4.1.0 +pytest-mock>=3.12.0 + +# Optional dependencies for full functionality +# ffmpeg-python>=0.2.0 # For media optimization (requires ffmpeg installed) diff --git a/skills/ai-multimodal/scripts/tests/test_document_converter.py b/skills/ai-multimodal/scripts/tests/test_document_converter.py new file mode 100644 index 0000000..be8c908 --- /dev/null +++ b/skills/ai-multimodal/scripts/tests/test_document_converter.py @@ -0,0 +1,299 @@ +""" +Tests for document_converter.py +""" + +import pytest +import sys +from pathlib import Path +from unittest.mock import Mock, patch, MagicMock + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import document_converter as dc + + +class TestEnvLoading: + """Test environment variable loading.""" + + @patch('document_converter.load_dotenv') + @patch('pathlib.Path.exists') + def test_load_env_files_success(self, mock_exists, mock_load_dotenv): + """Test successful .env file loading.""" + mock_exists.return_value = True + dc.load_env_files() + # Should be called for skill, skills, and claude dirs + assert mock_load_dotenv.call_count >= 1 + + @patch('document_converter.load_dotenv', None) + def test_load_env_files_no_dotenv(self): + """Test when dotenv is not available.""" + # Should not raise an error + dc.load_env_files() + + +class TestDependencyCheck: + """Test dependency checking.""" + + @patch('builtins.__import__') + def test_check_all_dependencies_available(self, mock_import): + """Test when all dependencies are available.""" + mock_import.return_value = Mock() + + deps = dc.check_dependencies() + + assert 'pypdf' in deps + assert 'markdown' in deps + assert 'pillow' in deps + + @patch('builtins.__import__') + def test_check_dependencies_missing(self, mock_import): + """Test when dependencies are missing.""" + def import_side_effect(name, *args, **kwargs): + if name == 'pypdf': + raise ImportError() + return Mock() + + mock_import.side_effect = import_side_effect + + # The function uses try/except, so we test the actual function + with patch('document_converter.sys.modules', {}): + # This is tricky to test due to import handling + pass + + +class TestPDFPageExtraction: + """Test PDF page extraction.""" + + @patch('pypdf.PdfReader') + @patch('pypdf.PdfWriter') + @patch('builtins.open', create=True) + def test_extract_single_page(self, mock_open, mock_writer_class, mock_reader_class): + """Test extracting a single page.""" + # Mock reader + mock_reader = Mock() + mock_page = Mock() + mock_reader.pages = [Mock(), mock_page, Mock()] + mock_reader_class.return_value = mock_reader + + # Mock writer + mock_writer = Mock() + mock_writer.pages = [mock_page] + mock_writer_class.return_value = mock_writer + + result = dc.extract_pdf_pages( + 'input.pdf', + 'output.pdf', + page_range='2', + verbose=False + ) + + assert result is True + mock_writer.add_page.assert_called_once_with(mock_page) + + @patch('pypdf.PdfReader') + @patch('pypdf.PdfWriter') + @patch('builtins.open', create=True) + def test_extract_page_range(self, mock_open, mock_writer_class, mock_reader_class): + """Test extracting a range of pages.""" + mock_reader = Mock() + mock_reader.pages = [Mock() for _ in range(10)] + mock_reader_class.return_value = mock_reader + + mock_writer = Mock() + mock_writer.pages = [] + mock_writer_class.return_value = mock_writer + + result = dc.extract_pdf_pages( + 'input.pdf', + 'output.pdf', + page_range='2-5', + verbose=False + ) + + assert result is True + assert mock_writer.add_page.call_count == 4 # Pages 2-5 (4 pages) + + def test_extract_pages_no_pypdf(self): + """Test page extraction without pypdf.""" + with patch.dict('sys.modules', {'pypdf': None}): + result = dc.extract_pdf_pages('input.pdf', 'output.pdf', '1-10') + assert result is False + + +class TestPDFOptimization: + """Test PDF optimization.""" + + @patch('pypdf.PdfReader') + @patch('pypdf.PdfWriter') + @patch('builtins.open', create=True) + @patch('pathlib.Path.stat') + def test_optimize_pdf_success(self, mock_stat, mock_open, mock_writer_class, mock_reader_class): + """Test successful PDF optimization.""" + # Mock reader + mock_reader = Mock() + mock_page = Mock() + mock_reader.pages = [mock_page, mock_page] + mock_reader_class.return_value = mock_reader + + # Mock writer + mock_writer = Mock() + mock_writer.pages = [mock_page, mock_page] + mock_writer_class.return_value = mock_writer + + # Mock file sizes + mock_stat.return_value.st_size = 1024 * 1024 + + result = dc.optimize_pdf('input.pdf', 'output.pdf', verbose=False) + + assert result is True + mock_page.compress_content_streams.assert_called() + + def test_optimize_pdf_no_pypdf(self): + """Test PDF optimization without pypdf.""" + with patch.dict('sys.modules', {'pypdf': None}): + result = dc.optimize_pdf('input.pdf', 'output.pdf') + assert result is False + + +class TestImageExtraction: + """Test image extraction from PDFs.""" + + @patch('pypdf.PdfReader') + @patch('PIL.Image') + @patch('pathlib.Path.mkdir') + @patch('builtins.open', create=True) + def test_extract_images_success(self, mock_open, mock_mkdir, mock_image, mock_reader_class): + """Test successful image extraction.""" + # Mock PDF reader + mock_reader = Mock() + mock_page = MagicMock() + + # Mock XObject with image + mock_obj = MagicMock() + mock_obj.__getitem__.side_effect = lambda k: { + '/Subtype': '/Image', + '/Width': 100, + '/Height': 100, + '/Filter': '/DCTDecode' + }[k] + mock_obj.get_data.return_value = b'image_data' + + mock_xobjects = MagicMock() + mock_xobjects.__iter__.return_value = ['img1'] + mock_xobjects.__getitem__.return_value = mock_obj + + mock_resources = MagicMock() + mock_resources.get_object.return_value = mock_xobjects + mock_page.__getitem__.side_effect = lambda k: { + '/Resources': {'/XObject': mock_resources} + }[k] + + mock_reader.pages = [mock_page] + mock_reader_class.return_value = mock_reader + + result = dc.extract_images_from_pdf('input.pdf', './output', verbose=False) + + assert len(result) > 0 + + def test_extract_images_no_dependencies(self): + """Test image extraction without required dependencies.""" + with patch.dict('sys.modules', {'pypdf': None}): + result = dc.extract_images_from_pdf('input.pdf', './output') + assert result == [] + + +class TestMarkdownConversion: + """Test Markdown to PDF conversion.""" + + @patch('markdown.markdown') + @patch('builtins.open', create=True) + @patch('subprocess.run') + @patch('pathlib.Path.unlink') + def test_convert_markdown_success(self, mock_unlink, mock_run, mock_open, mock_markdown): + """Test successful Markdown to PDF conversion.""" + mock_markdown.return_value = '

Test

' + + # Mock file reading and writing + mock_file = MagicMock() + mock_file.__enter__.return_value.read.return_value = '# Test' + mock_open.return_value = mock_file + + result = dc.convert_markdown_to_pdf('input.md', 'output.pdf', verbose=False) + + assert result is True + mock_run.assert_called_once() + + @patch('markdown.markdown') + @patch('builtins.open', create=True) + @patch('subprocess.run') + def test_convert_markdown_no_wkhtmltopdf(self, mock_run, mock_open, mock_markdown): + """Test Markdown conversion without wkhtmltopdf.""" + mock_markdown.return_value = '

Test

' + + mock_file = MagicMock() + mock_file.__enter__.return_value.read.return_value = '# Test' + mock_open.return_value = mock_file + + mock_run.side_effect = FileNotFoundError() + + result = dc.convert_markdown_to_pdf('input.md', 'output.pdf', verbose=False) + + assert result is False + + def test_convert_markdown_no_markdown_lib(self): + """Test Markdown conversion without markdown library.""" + with patch.dict('sys.modules', {'markdown': None}): + result = dc.convert_markdown_to_pdf('input.md', 'output.pdf') + assert result is False + + +class TestHTMLConversion: + """Test HTML to PDF conversion.""" + + @patch('subprocess.run') + def test_convert_html_success(self, mock_run): + """Test successful HTML to PDF conversion.""" + result = dc.convert_html_to_pdf('input.html', 'output.pdf', verbose=False) + + assert result is True + mock_run.assert_called_once() + + @patch('subprocess.run') + def test_convert_html_no_wkhtmltopdf(self, mock_run): + """Test HTML conversion without wkhtmltopdf.""" + mock_run.side_effect = FileNotFoundError() + + result = dc.convert_html_to_pdf('input.html', 'output.pdf', verbose=False) + + assert result is False + + +class TestIntegration: + """Integration tests.""" + + @patch('pathlib.Path.exists') + def test_file_not_found(self, mock_exists): + """Test handling of non-existent input file.""" + mock_exists.return_value = False + + # This would normally be tested via main() but we test the concept + assert not Path('nonexistent.pdf').exists() + + @patch('document_converter.check_dependencies') + def test_check_dependencies_integration(self, mock_check): + """Test dependency checking integration.""" + mock_check.return_value = { + 'pypdf': True, + 'markdown': True, + 'pillow': True + } + + deps = dc.check_dependencies() + + assert deps['pypdf'] is True + assert deps['markdown'] is True + assert deps['pillow'] is True + + +if __name__ == '__main__': + pytest.main([__file__, '-v', '--cov=document_converter', '--cov-report=term-missing']) diff --git a/skills/ai-multimodal/scripts/tests/test_gemini_batch_process.py b/skills/ai-multimodal/scripts/tests/test_gemini_batch_process.py new file mode 100644 index 0000000..7c90812 --- /dev/null +++ b/skills/ai-multimodal/scripts/tests/test_gemini_batch_process.py @@ -0,0 +1,362 @@ +""" +Tests for gemini_batch_process.py +""" + +import pytest +import sys +from pathlib import Path +from unittest.mock import Mock, patch, MagicMock + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import gemini_batch_process as gbp + + +class TestAPIKeyFinder: + """Test API key detection.""" + + def test_find_api_key_from_env(self, monkeypatch): + """Test finding API key from environment variable.""" + monkeypatch.setenv('GEMINI_API_KEY', 'test_key_123') + assert gbp.find_api_key() == 'test_key_123' + + @patch('gemini_batch_process.load_dotenv') + def test_find_api_key_not_found(self, mock_load_dotenv, monkeypatch): + """Test when API key is not found.""" + monkeypatch.delenv('GEMINI_API_KEY', raising=False) + # Mock load_dotenv to not actually load any files + mock_load_dotenv.return_value = None + assert gbp.find_api_key() is None + + +class TestMimeTypeDetection: + """Test MIME type detection.""" + + def test_audio_mime_types(self): + """Test audio file MIME types.""" + assert gbp.get_mime_type('test.mp3') == 'audio/mp3' + assert gbp.get_mime_type('test.wav') == 'audio/wav' + assert gbp.get_mime_type('test.aac') == 'audio/aac' + assert gbp.get_mime_type('test.flac') == 'audio/flac' + + def test_image_mime_types(self): + """Test image file MIME types.""" + assert gbp.get_mime_type('test.jpg') == 'image/jpeg' + assert gbp.get_mime_type('test.jpeg') == 'image/jpeg' + assert gbp.get_mime_type('test.png') == 'image/png' + assert gbp.get_mime_type('test.webp') == 'image/webp' + + def test_video_mime_types(self): + """Test video file MIME types.""" + assert gbp.get_mime_type('test.mp4') == 'video/mp4' + assert gbp.get_mime_type('test.mov') == 'video/quicktime' + assert gbp.get_mime_type('test.avi') == 'video/x-msvideo' + + def test_document_mime_types(self): + """Test document file MIME types.""" + assert gbp.get_mime_type('test.pdf') == 'application/pdf' + assert gbp.get_mime_type('test.txt') == 'text/plain' + + def test_unknown_mime_type(self): + """Test unknown file extension.""" + assert gbp.get_mime_type('test.xyz') == 'application/octet-stream' + + def test_case_insensitive(self): + """Test case-insensitive extension matching.""" + assert gbp.get_mime_type('TEST.MP3') == 'audio/mp3' + assert gbp.get_mime_type('Test.JPG') == 'image/jpeg' + + +class TestFileUpload: + """Test file upload functionality.""" + + @patch('gemini_batch_process.genai.Client') + def test_upload_file_success(self, mock_client_class): + """Test successful file upload.""" + # Mock client and file + mock_client = Mock() + mock_file = Mock() + mock_file.state.name = 'ACTIVE' + mock_file.name = 'test_file' + mock_client.files.upload.return_value = mock_file + + result = gbp.upload_file(mock_client, 'test.jpg', verbose=False) + + assert result == mock_file + mock_client.files.upload.assert_called_once_with(file='test.jpg') + + @patch('gemini_batch_process.genai.Client') + @patch('gemini_batch_process.time.sleep') + def test_upload_video_with_processing(self, mock_sleep, mock_client_class): + """Test video upload with processing wait.""" + mock_client = Mock() + + # First call: PROCESSING, second call: ACTIVE + mock_file_processing = Mock() + mock_file_processing.state.name = 'PROCESSING' + mock_file_processing.name = 'test_video' + + mock_file_active = Mock() + mock_file_active.state.name = 'ACTIVE' + mock_file_active.name = 'test_video' + + mock_client.files.upload.return_value = mock_file_processing + mock_client.files.get.return_value = mock_file_active + + result = gbp.upload_file(mock_client, 'test.mp4', verbose=False) + + assert result.state.name == 'ACTIVE' + + @patch('gemini_batch_process.genai.Client') + def test_upload_file_failed(self, mock_client_class): + """Test failed file upload.""" + mock_client = Mock() + mock_file = Mock() + mock_file.state.name = 'FAILED' + mock_client.files.upload.return_value = mock_file + mock_client.files.get.return_value = mock_file + + with pytest.raises(ValueError, match="File processing failed"): + gbp.upload_file(mock_client, 'test.mp4', verbose=False) + + +class TestProcessFile: + """Test file processing functionality.""" + + @patch('gemini_batch_process.genai.Client') + @patch('builtins.open', create=True) + @patch('pathlib.Path.stat') + def test_process_small_file_inline(self, mock_stat, mock_open, mock_client_class): + """Test processing small file with inline data.""" + # Mock small file + mock_stat.return_value.st_size = 10 * 1024 * 1024 # 10MB + + # Mock file content + mock_open.return_value.__enter__.return_value.read.return_value = b'test_data' + + # Mock client and response + mock_client = Mock() + mock_response = Mock() + mock_response.text = 'Test response' + mock_client.models.generate_content.return_value = mock_response + + result = gbp.process_file( + client=mock_client, + file_path='test.jpg', + prompt='Describe this image', + model='gemini-2.5-flash', + task='analyze', + format_output='text', + verbose=False + ) + + assert result['status'] == 'success' + assert result['response'] == 'Test response' + + @patch('gemini_batch_process.upload_file') + @patch('gemini_batch_process.genai.Client') + @patch('pathlib.Path.stat') + def test_process_large_file_api(self, mock_stat, mock_client_class, mock_upload): + """Test processing large file with File API.""" + # Mock large file + mock_stat.return_value.st_size = 50 * 1024 * 1024 # 50MB + + # Mock upload and response + mock_file = Mock() + mock_upload.return_value = mock_file + + mock_client = Mock() + mock_response = Mock() + mock_response.text = 'Test response' + mock_client.models.generate_content.return_value = mock_response + + result = gbp.process_file( + client=mock_client, + file_path='test.mp4', + prompt='Summarize this video', + model='gemini-2.5-flash', + task='analyze', + format_output='text', + verbose=False + ) + + assert result['status'] == 'success' + mock_upload.assert_called_once() + + @patch('gemini_batch_process.genai.Client') + @patch('builtins.open', create=True) + @patch('pathlib.Path.stat') + def test_process_file_error_handling(self, mock_stat, mock_open, mock_client_class): + """Test error handling in file processing.""" + mock_stat.return_value.st_size = 1024 + + # Mock file read + mock_file = MagicMock() + mock_file.__enter__.return_value.read.return_value = b'test_data' + mock_open.return_value = mock_file + + mock_client = Mock() + mock_client.models.generate_content.side_effect = Exception("API Error") + + result = gbp.process_file( + client=mock_client, + file_path='test.jpg', + prompt='Test', + model='gemini-2.5-flash', + task='analyze', + format_output='text', + verbose=False, + max_retries=1 + ) + + assert result['status'] == 'error' + assert 'API Error' in result['error'] + + @patch('gemini_batch_process.genai.Client') + @patch('builtins.open', create=True) + @patch('pathlib.Path.stat') + def test_image_generation_with_aspect_ratio(self, mock_stat, mock_open, mock_client_class): + """Test image generation with aspect ratio config.""" + mock_stat.return_value.st_size = 1024 + + # Mock file read + mock_file = MagicMock() + mock_file.__enter__.return_value.read.return_value = b'test' + mock_open.return_value = mock_file + + mock_client = Mock() + mock_response = Mock() + mock_response.candidates = [Mock()] + mock_response.candidates[0].content.parts = [ + Mock(inline_data=Mock(data=b'fake_image_data')) + ] + mock_client.models.generate_content.return_value = mock_response + + result = gbp.process_file( + client=mock_client, + file_path='test.txt', + prompt='Generate mountain landscape', + model='gemini-2.5-flash-image', + task='generate', + format_output='text', + aspect_ratio='16:9', + verbose=False + ) + + # Verify config was called with correct structure + call_args = mock_client.models.generate_content.call_args + config = call_args.kwargs.get('config') + assert config is not None + assert result['status'] == 'success' + assert 'generated_image' in result + + +class TestBatchProcessing: + """Test batch processing functionality.""" + + @patch('gemini_batch_process.find_api_key') + @patch('gemini_batch_process.process_file') + @patch('gemini_batch_process.genai.Client') + def test_batch_process_success(self, mock_client_class, mock_process, mock_find_key): + """Test successful batch processing.""" + mock_find_key.return_value = 'test_key' + mock_process.return_value = {'status': 'success', 'response': 'Test'} + + results = gbp.batch_process( + files=['test1.jpg', 'test2.jpg'], + prompt='Analyze', + model='gemini-2.5-flash', + task='analyze', + format_output='text', + verbose=False, + dry_run=False + ) + + assert len(results) == 2 + assert all(r['status'] == 'success' for r in results) + + @patch('gemini_batch_process.find_api_key') + def test_batch_process_no_api_key(self, mock_find_key): + """Test batch processing without API key.""" + mock_find_key.return_value = None + + with pytest.raises(SystemExit): + gbp.batch_process( + files=['test.jpg'], + prompt='Test', + model='gemini-2.5-flash', + task='analyze', + format_output='text', + verbose=False, + dry_run=False + ) + + @patch('gemini_batch_process.find_api_key') + def test_batch_process_dry_run(self, mock_find_key): + """Test dry run mode.""" + # API key not needed for dry run, but we mock it to avoid sys.exit + mock_find_key.return_value = 'test_key' + + results = gbp.batch_process( + files=['test1.jpg', 'test2.jpg'], + prompt='Test', + model='gemini-2.5-flash', + task='analyze', + format_output='text', + verbose=False, + dry_run=True + ) + + assert results == [] + + +class TestResultsSaving: + """Test results saving functionality.""" + + @patch('builtins.open', create=True) + @patch('json.dump') + def test_save_results_json(self, mock_json_dump, mock_open): + """Test saving results as JSON.""" + results = [ + {'file': 'test1.jpg', 'status': 'success', 'response': 'Test1'}, + {'file': 'test2.jpg', 'status': 'success', 'response': 'Test2'} + ] + + gbp.save_results(results, 'output.json', 'json') + + mock_json_dump.assert_called_once() + + @patch('builtins.open', create=True) + @patch('csv.DictWriter') + def test_save_results_csv(self, mock_csv_writer, mock_open): + """Test saving results as CSV.""" + results = [ + {'file': 'test1.jpg', 'status': 'success', 'response': 'Test1'}, + {'file': 'test2.jpg', 'status': 'success', 'response': 'Test2'} + ] + + gbp.save_results(results, 'output.csv', 'csv') + + # Verify CSV writer was used + mock_csv_writer.assert_called_once() + + @patch('builtins.open', create=True) + def test_save_results_markdown(self, mock_open): + """Test saving results as Markdown.""" + mock_file = MagicMock() + mock_open.return_value.__enter__.return_value = mock_file + + results = [ + {'file': 'test1.jpg', 'status': 'success', 'response': 'Test1'}, + {'file': 'test2.jpg', 'status': 'error', 'error': 'Failed'} + ] + + gbp.save_results(results, 'output.md', 'markdown') + + # Verify write was called + assert mock_file.write.call_count > 0 + + +if __name__ == '__main__': + pytest.main([__file__, '-v', '--cov=gemini_batch_process', '--cov-report=term-missing']) diff --git a/skills/ai-multimodal/scripts/tests/test_media_optimizer.py b/skills/ai-multimodal/scripts/tests/test_media_optimizer.py new file mode 100644 index 0000000..7a8c424 --- /dev/null +++ b/skills/ai-multimodal/scripts/tests/test_media_optimizer.py @@ -0,0 +1,373 @@ +""" +Tests for media_optimizer.py +""" + +import pytest +import sys +from pathlib import Path +from unittest.mock import Mock, patch, MagicMock +import json + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +import media_optimizer as mo + + +class TestEnvLoading: + """Test environment variable loading.""" + + @patch('media_optimizer.load_dotenv') + @patch('pathlib.Path.exists') + def test_load_env_files_success(self, mock_exists, mock_load_dotenv): + """Test successful .env file loading.""" + mock_exists.return_value = True + mo.load_env_files() + # Should be called for skill, skills, and claude dirs + assert mock_load_dotenv.call_count >= 1 + + @patch('media_optimizer.load_dotenv', None) + def test_load_env_files_no_dotenv(self): + """Test when dotenv is not available.""" + # Should not raise an error + mo.load_env_files() + + +class TestFFmpegCheck: + """Test ffmpeg availability checking.""" + + @patch('subprocess.run') + def test_ffmpeg_installed(self, mock_run): + """Test when ffmpeg is installed.""" + mock_run.return_value = Mock() + assert mo.check_ffmpeg() is True + + @patch('subprocess.run') + def test_ffmpeg_not_installed(self, mock_run): + """Test when ffmpeg is not installed.""" + mock_run.side_effect = FileNotFoundError() + assert mo.check_ffmpeg() is False + + @patch('subprocess.run') + def test_ffmpeg_error(self, mock_run): + """Test ffmpeg command error.""" + mock_run.side_effect = Exception("Error") + assert mo.check_ffmpeg() is False + + +class TestMediaInfo: + """Test media information extraction.""" + + @patch('media_optimizer.check_ffmpeg') + @patch('subprocess.run') + def test_get_video_info(self, mock_run, mock_check): + """Test extracting video information.""" + mock_check.return_value = True + + mock_result = Mock() + mock_result.stdout = json.dumps({ + 'format': { + 'size': '10485760', + 'duration': '120.5', + 'bit_rate': '691200' + }, + 'streams': [ + { + 'codec_type': 'video', + 'width': 1920, + 'height': 1080, + 'r_frame_rate': '30/1' + }, + { + 'codec_type': 'audio', + 'sample_rate': '48000', + 'channels': 2 + } + ] + }) + mock_run.return_value = mock_result + + info = mo.get_media_info('test.mp4') + + assert info['size'] == 10485760 + assert info['duration'] == 120.5 + assert info['width'] == 1920 + assert info['height'] == 1080 + assert info['sample_rate'] == 48000 + + @patch('media_optimizer.check_ffmpeg') + def test_get_media_info_no_ffmpeg(self, mock_check): + """Test when ffmpeg is not available.""" + mock_check.return_value = False + info = mo.get_media_info('test.mp4') + assert info == {} + + @patch('media_optimizer.check_ffmpeg') + @patch('subprocess.run') + def test_get_media_info_error(self, mock_run, mock_check): + """Test error handling in media info extraction.""" + mock_check.return_value = True + mock_run.side_effect = Exception("Error") + + info = mo.get_media_info('test.mp4') + assert info == {} + + +class TestVideoOptimization: + """Test video optimization functionality.""" + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + @patch('subprocess.run') + def test_optimize_video_success(self, mock_run, mock_info, mock_check): + """Test successful video optimization.""" + mock_check.return_value = True + mock_info.side_effect = [ + # Input info + { + 'size': 50 * 1024 * 1024, + 'duration': 120.0, + 'bit_rate': 3500000, + 'width': 1920, + 'height': 1080 + }, + # Output info + { + 'size': 25 * 1024 * 1024, + 'duration': 120.0, + 'width': 1920, + 'height': 1080 + } + ] + + result = mo.optimize_video( + 'input.mp4', + 'output.mp4', + quality=23, + verbose=False + ) + + assert result is True + mock_run.assert_called_once() + + @patch('media_optimizer.check_ffmpeg') + def test_optimize_video_no_ffmpeg(self, mock_check): + """Test video optimization without ffmpeg.""" + mock_check.return_value = False + + result = mo.optimize_video('input.mp4', 'output.mp4') + assert result is False + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + def test_optimize_video_no_info(self, mock_info, mock_check): + """Test video optimization when info cannot be read.""" + mock_check.return_value = True + mock_info.return_value = {} + + result = mo.optimize_video('input.mp4', 'output.mp4') + assert result is False + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + @patch('subprocess.run') + def test_optimize_video_with_target_size(self, mock_run, mock_info, mock_check): + """Test video optimization with target size.""" + mock_check.return_value = True + mock_info.side_effect = [ + {'size': 100 * 1024 * 1024, 'duration': 60.0, 'bit_rate': 3500000}, + {'size': 50 * 1024 * 1024, 'duration': 60.0} + ] + + result = mo.optimize_video( + 'input.mp4', + 'output.mp4', + target_size_mb=50, + verbose=False + ) + + assert result is True + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + @patch('subprocess.run') + def test_optimize_video_with_resolution(self, mock_run, mock_info, mock_check): + """Test video optimization with custom resolution.""" + mock_check.return_value = True + mock_info.side_effect = [ + {'size': 50 * 1024 * 1024, 'duration': 120.0, 'bit_rate': 3500000}, + {'size': 25 * 1024 * 1024, 'duration': 120.0} + ] + + result = mo.optimize_video( + 'input.mp4', + 'output.mp4', + resolution='1280x720', + verbose=False + ) + + assert result is True + + +class TestAudioOptimization: + """Test audio optimization functionality.""" + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + @patch('subprocess.run') + def test_optimize_audio_success(self, mock_run, mock_info, mock_check): + """Test successful audio optimization.""" + mock_check.return_value = True + mock_info.side_effect = [ + {'size': 10 * 1024 * 1024, 'duration': 300.0}, + {'size': 5 * 1024 * 1024, 'duration': 300.0} + ] + + result = mo.optimize_audio( + 'input.mp3', + 'output.m4a', + bitrate='64k', + verbose=False + ) + + assert result is True + mock_run.assert_called_once() + + @patch('media_optimizer.check_ffmpeg') + def test_optimize_audio_no_ffmpeg(self, mock_check): + """Test audio optimization without ffmpeg.""" + mock_check.return_value = False + + result = mo.optimize_audio('input.mp3', 'output.m4a') + assert result is False + + +class TestImageOptimization: + """Test image optimization functionality.""" + + @patch('PIL.Image.open') + @patch('pathlib.Path.stat') + def test_optimize_image_success(self, mock_stat, mock_image_open): + """Test successful image optimization.""" + # Mock image + mock_resized = Mock() + mock_resized.mode = 'RGB' + + mock_img = Mock() + mock_img.width = 3840 + mock_img.height = 2160 + mock_img.mode = 'RGB' + mock_img.resize.return_value = mock_resized + mock_image_open.return_value = mock_img + + # Mock file sizes + mock_stat.return_value.st_size = 5 * 1024 * 1024 + + result = mo.optimize_image( + 'input.jpg', + 'output.jpg', + max_width=1920, + quality=85, + verbose=False + ) + + assert result is True + # Since image is resized, save is called on the resized image + mock_resized.save.assert_called_once() + + @patch('PIL.Image.open') + @patch('pathlib.Path.stat') + def test_optimize_image_resize(self, mock_stat, mock_image_open): + """Test image resizing during optimization.""" + mock_img = Mock() + mock_img.width = 3840 + mock_img.height = 2160 + mock_img.mode = 'RGB' + mock_resized = Mock() + mock_img.resize.return_value = mock_resized + mock_image_open.return_value = mock_img + + mock_stat.return_value.st_size = 5 * 1024 * 1024 + + mo.optimize_image('input.jpg', 'output.jpg', max_width=1920, verbose=False) + + mock_img.resize.assert_called_once() + + @patch('PIL.Image.open') + @patch('pathlib.Path.stat') + def test_optimize_image_rgba_to_jpg(self, mock_stat, mock_image_open): + """Test converting RGBA to RGB for JPEG.""" + mock_img = Mock() + mock_img.width = 1920 + mock_img.height = 1080 + mock_img.mode = 'RGBA' + mock_img.split.return_value = [Mock(), Mock(), Mock(), Mock()] + mock_image_open.return_value = mock_img + + mock_stat.return_value.st_size = 1024 * 1024 + + with patch('PIL.Image.new') as mock_new: + mock_rgb = Mock() + mock_new.return_value = mock_rgb + + mo.optimize_image('input.png', 'output.jpg', verbose=False) + + mock_new.assert_called_once() + + def test_optimize_image_no_pillow(self): + """Test image optimization without Pillow.""" + with patch.dict('sys.modules', {'PIL': None}): + result = mo.optimize_image('input.jpg', 'output.jpg') + # Will fail to import but function handles it + assert result is False + + +class TestVideoSplitting: + """Test video splitting functionality.""" + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + @patch('subprocess.run') + @patch('pathlib.Path.mkdir') + def test_split_video_success(self, mock_mkdir, mock_run, mock_info, mock_check): + """Test successful video splitting.""" + mock_check.return_value = True + mock_info.return_value = {'duration': 7200.0} # 2 hours + + result = mo.split_video( + 'input.mp4', + './chunks', + chunk_duration=3600, # 1 hour chunks + verbose=False + ) + + # Duration 7200s / 3600s = 2, +1 for safety = 3 chunks + assert len(result) == 3 + assert mock_run.call_count == 3 + + @patch('media_optimizer.check_ffmpeg') + @patch('media_optimizer.get_media_info') + def test_split_video_short_duration(self, mock_info, mock_check): + """Test splitting video shorter than chunk duration.""" + mock_check.return_value = True + mock_info.return_value = {'duration': 1800.0} # 30 minutes + + result = mo.split_video( + 'input.mp4', + './chunks', + chunk_duration=3600, # 1 hour + verbose=False + ) + + assert result == ['input.mp4'] + + @patch('media_optimizer.check_ffmpeg') + def test_split_video_no_ffmpeg(self, mock_check): + """Test video splitting without ffmpeg.""" + mock_check.return_value = False + + result = mo.split_video('input.mp4', './chunks') + assert result == [] + + +if __name__ == '__main__': + pytest.main([__file__, '-v', '--cov=media_optimizer', '--cov-report=term-missing']) diff --git a/skills/better-auth/SKILL.md b/skills/better-auth/SKILL.md new file mode 100644 index 0000000..15ef839 --- /dev/null +++ b/skills/better-auth/SKILL.md @@ -0,0 +1,204 @@ +--- +name: better-auth +description: Implement authentication and authorization with Better Auth - a framework-agnostic TypeScript authentication framework. Features include email/password authentication with verification, OAuth providers (Google, GitHub, Discord, etc.), two-factor authentication (TOTP, SMS), passkeys/WebAuthn support, session management, role-based access control (RBAC), rate limiting, and database adapters. Use when adding authentication to applications, implementing OAuth flows, setting up 2FA/MFA, managing user sessions, configuring authorization rules, or building secure authentication systems for web applications. +license: MIT +version: 2.0.0 +--- + +# Better Auth Skill + +Better Auth is comprehensive, framework-agnostic authentication/authorization framework for TypeScript with built-in email/password, social OAuth, and powerful plugin ecosystem for advanced features. + +## When to Use + +- Implementing auth in TypeScript/JavaScript applications +- Adding email/password or social OAuth authentication +- Setting up 2FA, passkeys, magic links, advanced auth features +- Building multi-tenant apps with organization support +- Managing sessions and user lifecycle +- Working with any framework (Next.js, Nuxt, SvelteKit, Remix, Astro, Hono, Express, etc.) + +## Quick Start + +### Installation + +```bash +npm install better-auth +# or pnpm/yarn/bun add better-auth +``` + +### Environment Setup + +Create `.env`: +```env +BETTER_AUTH_SECRET= +BETTER_AUTH_URL=http://localhost:3000 +``` + +### Basic Server Setup + +Create `auth.ts` (root, lib/, utils/, or under src/app/server/): + +```ts +import { betterAuth } from "better-auth"; + +export const auth = betterAuth({ + database: { + // See references/database-integration.md + }, + emailAndPassword: { + enabled: true, + autoSignIn: true + }, + socialProviders: { + github: { + clientId: process.env.GITHUB_CLIENT_ID!, + clientSecret: process.env.GITHUB_CLIENT_SECRET!, + } + } +}); +``` + +### Database Schema + +```bash +npx @better-auth/cli generate # Generate schema/migrations +npx @better-auth/cli migrate # Apply migrations (Kysely only) +``` + +### Mount API Handler + +**Next.js App Router:** +```ts +// app/api/auth/[...all]/route.ts +import { auth } from "@/lib/auth"; +import { toNextJsHandler } from "better-auth/next-js"; + +export const { POST, GET } = toNextJsHandler(auth); +``` + +**Other frameworks:** See references/email-password-auth.md#framework-setup + +### Client Setup + +Create `auth-client.ts`: + +```ts +import { createAuthClient } from "better-auth/client"; + +export const authClient = createAuthClient({ + baseURL: process.env.NEXT_PUBLIC_BETTER_AUTH_URL || "http://localhost:3000" +}); +``` + +### Basic Usage + +```ts +// Sign up +await authClient.signUp.email({ + email: "user@example.com", + password: "secure123", + name: "John Doe" +}); + +// Sign in +await authClient.signIn.email({ + email: "user@example.com", + password: "secure123" +}); + +// OAuth +await authClient.signIn.social({ provider: "github" }); + +// Session +const { data: session } = authClient.useSession(); // React/Vue/Svelte +const { data: session } = await authClient.getSession(); // Vanilla JS +``` + +## Feature Selection Matrix + +| Feature | Plugin Required | Use Case | Reference | +|---------|----------------|----------|-----------| +| Email/Password | No (built-in) | Basic auth | [email-password-auth.md](./references/email-password-auth.md) | +| OAuth (GitHub, Google, etc.) | No (built-in) | Social login | [oauth-providers.md](./references/oauth-providers.md) | +| Email Verification | No (built-in) | Verify email addresses | [email-password-auth.md](./references/email-password-auth.md#email-verification) | +| Password Reset | No (built-in) | Forgot password flow | [email-password-auth.md](./references/email-password-auth.md#password-reset) | +| Two-Factor Auth (2FA/TOTP) | Yes (`twoFactor`) | Enhanced security | [advanced-features.md](./references/advanced-features.md#two-factor-authentication) | +| Passkeys/WebAuthn | Yes (`passkey`) | Passwordless auth | [advanced-features.md](./references/advanced-features.md#passkeys-webauthn) | +| Magic Link | Yes (`magicLink`) | Email-based login | [advanced-features.md](./references/advanced-features.md#magic-link) | +| Username Auth | Yes (`username`) | Username login | [email-password-auth.md](./references/email-password-auth.md#username-authentication) | +| Organizations/Multi-tenant | Yes (`organization`) | Team/org features | [advanced-features.md](./references/advanced-features.md#organizations) | +| Rate Limiting | No (built-in) | Prevent abuse | [advanced-features.md](./references/advanced-features.md#rate-limiting) | +| Session Management | No (built-in) | User sessions | [advanced-features.md](./references/advanced-features.md#session-management) | + +## Auth Method Selection Guide + +**Choose Email/Password when:** +- Building standard web app with traditional auth +- Need full control over user credentials +- Targeting users who prefer email-based accounts + +**Choose OAuth when:** +- Want quick signup with minimal friction +- Users already have social accounts +- Need access to social profile data + +**Choose Passkeys when:** +- Want passwordless experience +- Targeting modern browsers/devices +- Security is top priority + +**Choose Magic Link when:** +- Want passwordless without WebAuthn complexity +- Targeting email-first users +- Need temporary access links + +**Combine Multiple Methods when:** +- Want flexibility for different user preferences +- Building enterprise apps with various auth requirements +- Need progressive enhancement (start simple, add more options) + +## Core Architecture + +Better Auth uses client-server architecture: +1. **Server** (`better-auth`): Handles auth logic, database ops, API routes +2. **Client** (`better-auth/client`): Provides hooks/methods for frontend +3. **Plugins**: Extend both server/client functionality + +## Implementation Checklist + +- [ ] Install `better-auth` package +- [ ] Set environment variables (SECRET, URL) +- [ ] Create auth server instance with database config +- [ ] Run schema migration (`npx @better-auth/cli generate`) +- [ ] Mount API handler in framework +- [ ] Create client instance +- [ ] Implement sign-up/sign-in UI +- [ ] Add session management to components +- [ ] Set up protected routes/middleware +- [ ] Add plugins as needed (regenerate schema after) +- [ ] Test complete auth flow +- [ ] Configure email sending (verification/reset) +- [ ] Enable rate limiting for production +- [ ] Set up error handling + +## Reference Documentation + +### Core Authentication +- [Email/Password Authentication](./references/email-password-auth.md) - Email/password setup, verification, password reset, username auth +- [OAuth Providers](./references/oauth-providers.md) - Social login setup, provider configuration, token management +- [Database Integration](./references/database-integration.md) - Database adapters, schema setup, migrations + +### Advanced Features +- [Advanced Features](./references/advanced-features.md) - 2FA/MFA, passkeys, magic links, organizations, rate limiting, session management + +## Scripts + +- `scripts/better_auth_init.py` - Initialize Better Auth configuration with interactive setup + +## Resources + +- Docs: https://www.better-auth.com/docs +- GitHub: https://github.com/better-auth/better-auth +- Plugins: https://www.better-auth.com/docs/plugins +- Examples: https://www.better-auth.com/docs/examples diff --git a/skills/better-auth/references/advanced-features.md b/skills/better-auth/references/advanced-features.md new file mode 100644 index 0000000..59d607b --- /dev/null +++ b/skills/better-auth/references/advanced-features.md @@ -0,0 +1,553 @@ +# Advanced Features + +Better Auth plugins extend functionality beyond basic authentication. + +## Two-Factor Authentication + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { twoFactor } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + twoFactor({ + issuer: "YourAppName", // TOTP issuer name + otpOptions: { + period: 30, // OTP validity period (seconds) + digits: 6, // OTP length + } + }) + ] +}); +``` + +### Client Setup + +```ts +import { createAuthClient } from "better-auth/client"; +import { twoFactorClient } from "better-auth/client/plugins"; + +export const authClient = createAuthClient({ + plugins: [ + twoFactorClient({ + twoFactorPage: "/two-factor", // Redirect to 2FA verification page + redirect: true // Auto-redirect if 2FA required + }) + ] +}); +``` + +### Enable 2FA for User + +```ts +// Enable TOTP +const { data } = await authClient.twoFactor.enable({ + password: "userPassword" // Verify user identity +}); + +// data contains QR code URI for authenticator app +const qrCodeUri = data.totpURI; +const backupCodes = data.backupCodes; // Save these securely +``` + +### Verify TOTP Code + +```ts +await authClient.twoFactor.verifyTOTP({ + code: "123456", + trustDevice: true // Skip 2FA on this device for 30 days +}); +``` + +### Disable 2FA + +```ts +await authClient.twoFactor.disable({ + password: "userPassword" +}); +``` + +### Backup Codes + +```ts +// Generate new backup codes +const { data } = await authClient.twoFactor.generateBackupCodes({ + password: "userPassword" +}); + +// Use backup code instead of TOTP +await authClient.twoFactor.verifyBackupCode({ + code: "backup-code-123" +}); +``` + +## Passkeys (WebAuthn) + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { passkey } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + passkey({ + rpName: "YourApp", // Relying Party name + rpID: "yourdomain.com" // Your domain + }) + ] +}); +``` + +### Client Setup + +```ts +import { createAuthClient } from "better-auth/client"; +import { passkeyClient } from "better-auth/client/plugins"; + +export const authClient = createAuthClient({ + plugins: [passkeyClient()] +}); +``` + +### Register Passkey + +```ts +// User must be authenticated first +await authClient.passkey.register({ + name: "My Laptop" // Optional: name for this passkey +}); +``` + +### Sign In with Passkey + +```ts +await authClient.passkey.signIn(); +``` + +### List User Passkeys + +```ts +const { data } = await authClient.passkey.list(); +// data contains array of registered passkeys +``` + +### Delete Passkey + +```ts +await authClient.passkey.delete({ + id: "passkey-id" +}); +``` + +## Magic Link + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { magicLink } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + magicLink({ + sendMagicLink: async ({ email, url, token }) => { + await sendEmail({ + to: email, + subject: "Sign in to YourApp", + html: `Click here to sign in.` + }); + }, + expiresIn: 300, // Link expires in 5 minutes (seconds) + }) + ] +}); +``` + +### Client Setup + +```ts +import { createAuthClient } from "better-auth/client"; +import { magicLinkClient } from "better-auth/client/plugins"; + +export const authClient = createAuthClient({ + plugins: [magicLinkClient()] +}); +``` + +### Send Magic Link + +```ts +await authClient.magicLink.sendMagicLink({ + email: "user@example.com", + callbackURL: "/dashboard" +}); +``` + +### Verify Magic Link + +```ts +// Called automatically when user clicks link +// Token in URL query params handled by Better Auth +await authClient.magicLink.verify({ + token: "token-from-url" +}); +``` + +## Organizations (Multi-Tenancy) + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { organization } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + organization({ + allowUserToCreateOrganization: true, + organizationLimit: 5, // Max orgs per user + creatorRole: "owner" // Role for org creator + }) + ] +}); +``` + +### Client Setup + +```ts +import { createAuthClient } from "better-auth/client"; +import { organizationClient } from "better-auth/client/plugins"; + +export const authClient = createAuthClient({ + plugins: [organizationClient()] +}); +``` + +### Create Organization + +```ts +await authClient.organization.create({ + name: "Acme Corp", + slug: "acme", // Unique slug + metadata: { + industry: "Technology" + } +}); +``` + +### Invite Members + +```ts +await authClient.organization.inviteMember({ + organizationId: "org-id", + email: "user@example.com", + role: "member", // owner, admin, member + message: "Join our team!" // Optional +}); +``` + +### Accept Invitation + +```ts +await authClient.organization.acceptInvitation({ + invitationId: "invitation-id" +}); +``` + +### List Organizations + +```ts +const { data } = await authClient.organization.list(); +// Returns user's organizations +``` + +### Update Member Role + +```ts +await authClient.organization.updateMemberRole({ + organizationId: "org-id", + userId: "user-id", + role: "admin" +}); +``` + +### Remove Member + +```ts +await authClient.organization.removeMember({ + organizationId: "org-id", + userId: "user-id" +}); +``` + +### Delete Organization + +```ts +await authClient.organization.delete({ + organizationId: "org-id" +}); +``` + +## Session Management + +### Configure Session Expiration + +```ts +export const auth = betterAuth({ + session: { + expiresIn: 60 * 60 * 24 * 7, // 7 days (seconds) + updateAge: 60 * 60 * 24, // Update session every 24 hours + cookieCache: { + enabled: true, + maxAge: 5 * 60 // Cache for 5 minutes + } + } +}); +``` + +### Server-Side Session + +```ts +// Next.js +import { auth } from "@/lib/auth"; +import { headers } from "next/headers"; + +const session = await auth.api.getSession({ + headers: await headers() +}); + +if (!session) { + // Not authenticated +} +``` + +### Client-Side Session + +```tsx +// React +import { authClient } from "@/lib/auth-client"; + +function UserProfile() { + const { data: session, isPending, error } = authClient.useSession(); + + if (isPending) return
Loading...
; + if (error) return
Error
; + if (!session) return
Not logged in
; + + return
Hello, {session.user.name}!
; +} +``` + +### List Active Sessions + +```ts +const { data: sessions } = await authClient.listSessions(); +// Returns all active sessions for current user +``` + +### Revoke Session + +```ts +await authClient.revokeSession({ + sessionId: "session-id" +}); +``` + +### Revoke All Sessions + +```ts +await authClient.revokeAllSessions(); +``` + +## Rate Limiting + +### Server Configuration + +```ts +export const auth = betterAuth({ + rateLimit: { + enabled: true, + window: 60, // Time window in seconds + max: 10, // Max requests per window + storage: "memory", // "memory" or "database" + customRules: { + "/api/auth/sign-in": { + window: 60, + max: 5 // Stricter limit for sign-in + }, + "/api/auth/sign-up": { + window: 3600, + max: 3 // 3 signups per hour + } + } + } +}); +``` + +### Custom Rate Limiter + +```ts +import { betterAuth } from "better-auth"; + +export const auth = betterAuth({ + rateLimit: { + enabled: true, + customLimiter: async ({ request, limit }) => { + // Custom rate limiting logic + const ip = request.headers.get("x-forwarded-for"); + const key = `ratelimit:${ip}`; + + // Use Redis, etc. + const count = await redis.incr(key); + if (count === 1) { + await redis.expire(key, limit.window); + } + + if (count > limit.max) { + throw new Error("Rate limit exceeded"); + } + } + } +}); +``` + +## Anonymous Sessions + +Track users before they sign up. + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { anonymous } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [anonymous()] +}); +``` + +### Client Usage + +```ts +// Create anonymous session +const { data } = await authClient.signIn.anonymous(); + +// Convert to full account +await authClient.signUp.email({ + email: "user@example.com", + password: "password123", + linkAnonymousSession: true // Link anonymous data +}); +``` + +## Email OTP + +One-time password via email (passwordless). + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { emailOTP } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + emailOTP({ + sendVerificationOTP: async ({ email, otp }) => { + await sendEmail({ + to: email, + subject: "Your verification code", + text: `Your code is: ${otp}` + }); + }, + expiresIn: 300, // 5 minutes + length: 6 // OTP length + }) + ] +}); +``` + +### Client Usage + +```ts +// Send OTP to email +await authClient.emailOTP.sendOTP({ + email: "user@example.com" +}); + +// Verify OTP +await authClient.emailOTP.verifyOTP({ + email: "user@example.com", + otp: "123456" +}); +``` + +## Phone Number Authentication + +Requires phone number plugin. + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { phoneNumber } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + phoneNumber({ + sendOTP: async ({ phoneNumber, otp }) => { + // Use Twilio, AWS SNS, etc. + await sendSMS(phoneNumber, `Your code: ${otp}`); + } + }) + ] +}); +``` + +### Client Usage + +```ts +// Sign up with phone +await authClient.signUp.phoneNumber({ + phoneNumber: "+1234567890", + password: "password123" +}); + +// Send OTP +await authClient.phoneNumber.sendOTP({ + phoneNumber: "+1234567890" +}); + +// Verify OTP +await authClient.phoneNumber.verifyOTP({ + phoneNumber: "+1234567890", + otp: "123456" +}); +``` + +## Best Practices + +1. **2FA**: Offer 2FA as optional, make mandatory for admin users +2. **Passkeys**: Implement as progressive enhancement (fallback to password) +3. **Magic Links**: Set short expiration (5-15 minutes) +4. **Organizations**: Implement RBAC for org permissions +5. **Sessions**: Use short expiration for sensitive apps +6. **Rate Limiting**: Enable in production, adjust limits based on usage +7. **Anonymous Sessions**: Clean up old anonymous sessions periodically +8. **Backup Codes**: Force users to save backup codes before enabling 2FA +9. **Multi-Device**: Allow users to manage trusted devices +10. **Audit Logs**: Track sensitive operations (role changes, 2FA changes) + +## Regenerate Schema After Plugins + +After adding any plugin: + +```bash +npx @better-auth/cli generate +npx @better-auth/cli migrate # if using Kysely +``` + +Or manually apply migrations for your ORM (Drizzle, Prisma). diff --git a/skills/better-auth/references/database-integration.md b/skills/better-auth/references/database-integration.md new file mode 100644 index 0000000..2a13565 --- /dev/null +++ b/skills/better-auth/references/database-integration.md @@ -0,0 +1,577 @@ +# Database Integration + +Better Auth supports multiple databases and ORMs for flexible data persistence. + +## Supported Databases + +- SQLite +- PostgreSQL +- MySQL/MariaDB +- MongoDB +- Any database with adapter support + +## Direct Database Connection + +### SQLite + +```ts +import { betterAuth } from "better-auth"; +import Database from "better-sqlite3"; + +export const auth = betterAuth({ + database: new Database("./sqlite.db"), + // or + database: new Database(":memory:") // In-memory for testing +}); +``` + +### PostgreSQL + +```ts +import { betterAuth } from "better-auth"; +import { Pool } from "pg"; + +const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + // or explicit config + host: "localhost", + port: 5432, + user: "postgres", + password: "password", + database: "myapp" +}); + +export const auth = betterAuth({ + database: pool +}); +``` + +### MySQL + +```ts +import { betterAuth } from "better-auth"; +import { createPool } from "mysql2/promise"; + +const pool = createPool({ + host: "localhost", + user: "root", + password: "password", + database: "myapp", + waitForConnections: true, + connectionLimit: 10 +}); + +export const auth = betterAuth({ + database: pool +}); +``` + +## ORM Adapters + +### Drizzle ORM + +**Install:** +```bash +npm install drizzle-orm better-auth +``` + +**Setup:** +```ts +import { betterAuth } from "better-auth"; +import { drizzleAdapter } from "better-auth/adapters/drizzle"; +import { drizzle } from "drizzle-orm/node-postgres"; +import { Pool } from "pg"; + +const pool = new Pool({ + connectionString: process.env.DATABASE_URL +}); + +const db = drizzle(pool); + +export const auth = betterAuth({ + database: drizzleAdapter(db, { + provider: "pg", // "pg" | "mysql" | "sqlite" + schema: { + // Optional: custom table names + user: "users", + session: "sessions", + account: "accounts", + verification: "verifications" + } + }) +}); +``` + +**Generate Schema:** +```bash +npx @better-auth/cli generate --adapter drizzle +``` + +### Prisma + +**Install:** +```bash +npm install @prisma/client better-auth +``` + +**Setup:** +```ts +import { betterAuth } from "better-auth"; +import { prismaAdapter } from "better-auth/adapters/prisma"; +import { PrismaClient } from "@prisma/client"; + +const prisma = new PrismaClient(); + +export const auth = betterAuth({ + database: prismaAdapter(prisma, { + provider: "postgresql", // "postgresql" | "mysql" | "sqlite" + }) +}); +``` + +**Generate Schema:** +```bash +npx @better-auth/cli generate --adapter prisma +``` + +**Apply to Prisma:** +```bash +# Add generated schema to schema.prisma +npx prisma migrate dev --name init +npx prisma generate +``` + +### Kysely + +**Install:** +```bash +npm install kysely better-auth +``` + +**Setup:** +```ts +import { betterAuth } from "better-auth"; +import { kyselyAdapter } from "better-auth/adapters/kysely"; +import { Kysely, PostgresDialect } from "kysely"; +import { Pool } from "pg"; + +const db = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool({ + connectionString: process.env.DATABASE_URL + }) + }) +}); + +export const auth = betterAuth({ + database: kyselyAdapter(db, { + provider: "pg" + }) +}); +``` + +**Auto-migrate with Kysely:** +```bash +npx @better-auth/cli migrate --adapter kysely +``` + +### MongoDB + +**Install:** +```bash +npm install mongodb better-auth +``` + +**Setup:** +```ts +import { betterAuth } from "better-auth"; +import { mongodbAdapter } from "better-auth/adapters/mongodb"; +import { MongoClient } from "mongodb"; + +const client = new MongoClient(process.env.MONGODB_URI!); +await client.connect(); + +export const auth = betterAuth({ + database: mongodbAdapter(client, { + databaseName: "myapp" + }) +}); +``` + +**Generate Collections:** +```bash +npx @better-auth/cli generate --adapter mongodb +``` + +## Core Database Schema + +Better Auth requires these core tables/collections: + +### User Table + +```sql +CREATE TABLE user ( + id TEXT PRIMARY KEY, + email TEXT UNIQUE NOT NULL, + emailVerified BOOLEAN DEFAULT FALSE, + name TEXT, + image TEXT, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +### Session Table + +```sql +CREATE TABLE session ( + id TEXT PRIMARY KEY, + userId TEXT NOT NULL, + expiresAt TIMESTAMP NOT NULL, + ipAddress TEXT, + userAgent TEXT, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (userId) REFERENCES user(id) ON DELETE CASCADE +); +``` + +### Account Table + +```sql +CREATE TABLE account ( + id TEXT PRIMARY KEY, + userId TEXT NOT NULL, + accountId TEXT NOT NULL, + providerId TEXT NOT NULL, + accessToken TEXT, + refreshToken TEXT, + expiresAt TIMESTAMP, + scope TEXT, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (userId) REFERENCES user(id) ON DELETE CASCADE, + UNIQUE(providerId, accountId) +); +``` + +### Verification Table + +```sql +CREATE TABLE verification ( + id TEXT PRIMARY KEY, + identifier TEXT NOT NULL, + value TEXT NOT NULL, + expiresAt TIMESTAMP NOT NULL, + createdAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + updatedAt TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); +``` + +## Schema Generation + +### Using CLI + +```bash +# Generate schema files +npx @better-auth/cli generate + +# Specify adapter +npx @better-auth/cli generate --adapter drizzle +npx @better-auth/cli generate --adapter prisma + +# Specify output +npx @better-auth/cli generate --output ./db/schema.ts +``` + +### Auto-migrate (Kysely only) + +```bash +npx @better-auth/cli migrate +``` + +For other ORMs, apply generated schema manually. + +## Custom Fields + +Add custom fields to user table: + +```ts +export const auth = betterAuth({ + user: { + additionalFields: { + role: { + type: "string", + required: false, + defaultValue: "user" + }, + phoneNumber: { + type: "string", + required: false + }, + subscriptionTier: { + type: "string", + required: false + } + } + } +}); +``` + +After adding fields: +```bash +npx @better-auth/cli generate +``` + +Update user with custom fields: +```ts +await authClient.updateUser({ + role: "admin", + phoneNumber: "+1234567890" +}); +``` + +## Plugin Schema Extensions + +Plugins add their own tables/fields. Regenerate schema after adding plugins: + +```bash +npx @better-auth/cli generate +``` + +### Two-Factor Plugin Tables + +- `twoFactor`: Stores TOTP secrets, backup codes + +### Passkey Plugin Tables + +- `passkey`: Stores WebAuthn credentials + +### Organization Plugin Tables + +- `organization`: Organization data +- `member`: Organization members +- `invitation`: Pending invitations + +## Migration Strategies + +### Development + +```bash +# Generate schema +npx @better-auth/cli generate + +# Apply migrations (Kysely) +npx @better-auth/cli migrate + +# Or manual (Prisma) +npx prisma migrate dev + +# Or manual (Drizzle) +npx drizzle-kit push +``` + +### Production + +```bash +# Review generated migration +npx @better-auth/cli generate + +# Test in staging +# Apply to production with your ORM's migration tool + +# Prisma +npx prisma migrate deploy + +# Drizzle +npx drizzle-kit push + +# Kysely +npx @better-auth/cli migrate +``` + +## Connection Pooling + +### PostgreSQL + +```ts +import { Pool } from "pg"; + +const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + max: 20, // Max connections + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 2000, +}); +``` + +### MySQL + +```ts +import { createPool } from "mysql2/promise"; + +const pool = createPool({ + connectionString: process.env.DATABASE_URL, + waitForConnections: true, + connectionLimit: 10, + queueLimit: 0 +}); +``` + +## Database URLs + +### PostgreSQL + +```env +DATABASE_URL=postgresql://user:password@localhost:5432/dbname +# Or with connection params +DATABASE_URL=postgresql://user:password@localhost:5432/dbname?schema=public&connection_limit=10 +``` + +### MySQL + +```env +DATABASE_URL=mysql://user:password@localhost:3306/dbname +``` + +### SQLite + +```env +DATABASE_URL=file:./dev.db +# Or in-memory +DATABASE_URL=:memory: +``` + +### MongoDB + +```env +MONGODB_URI=mongodb://localhost:27017/dbname +# Or Atlas +MONGODB_URI=mongodb+srv://user:password@cluster.mongodb.net/dbname +``` + +## Performance Optimization + +### Indexes + +Better Auth CLI auto-generates essential indexes: +- `user.email` (unique) +- `session.userId` +- `account.userId` +- `account.providerId, accountId` (unique) + +Add custom indexes for performance: +```sql +CREATE INDEX idx_session_expires ON session(expiresAt); +CREATE INDEX idx_user_created ON user(createdAt); +``` + +### Query Optimization + +```ts +// Use connection pooling +// Enable query caching where applicable +// Monitor slow queries + +export const auth = betterAuth({ + advanced: { + defaultCookieAttributes: { + sameSite: "lax", + secure: true, + httpOnly: true + } + } +}); +``` + +## Backup Strategies + +### PostgreSQL + +```bash +# Backup +pg_dump dbname > backup.sql + +# Restore +psql dbname < backup.sql +``` + +### MySQL + +```bash +# Backup +mysqldump -u root -p dbname > backup.sql + +# Restore +mysql -u root -p dbname < backup.sql +``` + +### SQLite + +```bash +# Copy file +cp dev.db dev.db.backup + +# Or use backup command +sqlite3 dev.db ".backup backup.db" +``` + +### MongoDB + +```bash +# Backup +mongodump --db=dbname --out=./backup + +# Restore +mongorestore --db=dbname ./backup/dbname +``` + +## Best Practices + +1. **Environment Variables**: Store credentials in env vars, never commit +2. **Connection Pooling**: Use pools for PostgreSQL/MySQL in production +3. **Migrations**: Use ORM migration tools, not raw SQL in production +4. **Indexes**: Add indexes for frequently queried fields +5. **Backups**: Automate daily backups in production +6. **SSL**: Use SSL/TLS for database connections in production +7. **Schema Sync**: Keep schema in sync across environments +8. **Testing**: Use separate database for tests (in-memory SQLite ideal) +9. **Monitoring**: Monitor query performance and connection pool usage +10. **Cleanup**: Periodically clean expired sessions/verifications + +## Troubleshooting + +### Connection Errors + +```ts +// Add connection timeout +const pool = new Pool({ + connectionString: process.env.DATABASE_URL, + connectionTimeoutMillis: 5000 +}); +``` + +### Schema Mismatch + +```bash +# Regenerate schema +npx @better-auth/cli generate + +# Apply migrations +# For Prisma: npx prisma migrate dev +# For Drizzle: npx drizzle-kit push +``` + +### Migration Failures + +- Check database credentials +- Verify database server is running +- Check for schema conflicts +- Review migration SQL manually + +### Performance Issues + +- Add indexes on foreign keys +- Enable connection pooling +- Monitor slow queries +- Consider read replicas for heavy read workloads diff --git a/skills/better-auth/references/email-password-auth.md b/skills/better-auth/references/email-password-auth.md new file mode 100644 index 0000000..eff3ec1 --- /dev/null +++ b/skills/better-auth/references/email-password-auth.md @@ -0,0 +1,416 @@ +# Email/Password Authentication + +Email/password is built-in auth method in Better Auth. No plugins required for basic functionality. + +## Server Configuration + +### Basic Setup + +```ts +import { betterAuth } from "better-auth"; + +export const auth = betterAuth({ + emailAndPassword: { + enabled: true, + autoSignIn: true, // Auto sign-in after signup (default: true) + requireEmailVerification: false, // Require email verification before login + sendResetPasswordToken: async ({ user, url }) => { + // Send password reset email + await sendEmail(user.email, url); + } + } +}); +``` + +### Custom Password Requirements + +```ts +export const auth = betterAuth({ + emailAndPassword: { + enabled: true, + password: { + minLength: 8, + requireUppercase: true, + requireLowercase: true, + requireNumbers: true, + requireSpecialChars: true + } + } +}); +``` + +## Client Usage + +### Sign Up + +```ts +import { authClient } from "@/lib/auth-client"; + +const { data, error } = await authClient.signUp.email({ + email: "user@example.com", + password: "securePassword123", + name: "John Doe", + image: "https://example.com/avatar.jpg", // optional + callbackURL: "/dashboard" // optional +}, { + onSuccess: (ctx) => { + // ctx.data contains user and session + console.log("User created:", ctx.data.user); + }, + onError: (ctx) => { + alert(ctx.error.message); + } +}); +``` + +### Sign In + +```ts +const { data, error } = await authClient.signIn.email({ + email: "user@example.com", + password: "securePassword123", + callbackURL: "/dashboard", + rememberMe: true // default: true +}, { + onSuccess: () => { + // redirect or update UI + }, + onError: (ctx) => { + console.error(ctx.error.message); + } +}); +``` + +### Sign Out + +```ts +await authClient.signOut({ + fetchOptions: { + onSuccess: () => { + router.push("/login"); + } + } +}); +``` + +## Email Verification + +### Server Setup + +```ts +export const auth = betterAuth({ + emailVerification: { + sendVerificationEmail: async ({ user, url, token }) => { + // Send verification email + await sendEmail({ + to: user.email, + subject: "Verify your email", + html: `Click here to verify your email.` + }); + }, + sendOnSignUp: true, // Send verification email on signup + autoSignInAfterVerification: true // Auto sign-in after verification + }, + emailAndPassword: { + enabled: true, + requireEmailVerification: true // Require verification before login + } +}); +``` + +### Client Usage + +```ts +// Send verification email +await authClient.sendVerificationEmail({ + email: "user@example.com", + callbackURL: "/verify-success" +}); + +// Verify email with token +await authClient.verifyEmail({ + token: "verification-token-from-email" +}); +``` + +## Password Reset Flow + +### Server Setup + +```ts +export const auth = betterAuth({ + emailAndPassword: { + enabled: true, + sendResetPasswordToken: async ({ user, url, token }) => { + await sendEmail({ + to: user.email, + subject: "Reset your password", + html: `Click here to reset your password.` + }); + } + } +}); +``` + +### Client Flow + +```ts +// Step 1: Request password reset +await authClient.forgetPassword({ + email: "user@example.com", + redirectTo: "/reset-password" +}); + +// Step 2: Reset password with token +await authClient.resetPassword({ + token: "reset-token-from-email", + password: "newSecurePassword123" +}); +``` + +### Change Password (Authenticated) + +```ts +await authClient.changePassword({ + currentPassword: "oldPassword123", + newPassword: "newPassword456", + revokeOtherSessions: true // Optional: logout other sessions +}); +``` + +## Username Authentication + +Requires `username` plugin for username-based auth. + +### Server Setup + +```ts +import { betterAuth } from "better-auth"; +import { username } from "better-auth/plugins"; + +export const auth = betterAuth({ + plugins: [ + username({ + // Allow sign in with username or email + allowUsernameOrEmail: true + }) + ] +}); +``` + +### Client Setup + +```ts +import { createAuthClient } from "better-auth/client"; +import { usernameClient } from "better-auth/client/plugins"; + +export const authClient = createAuthClient({ + plugins: [usernameClient()] +}); +``` + +### Client Usage + +```ts +// Sign up with username +await authClient.signUp.username({ + username: "johndoe", + password: "securePassword123", + email: "john@example.com", // optional + name: "John Doe" +}); + +// Sign in with username +await authClient.signIn.username({ + username: "johndoe", + password: "securePassword123" +}); + +// Sign in with username or email (if allowUsernameOrEmail: true) +await authClient.signIn.username({ + username: "johndoe", // or "john@example.com" + password: "securePassword123" +}); +``` + +## Framework Setup + +### Next.js (App Router) + +```ts +// app/api/auth/[...all]/route.ts +import { auth } from "@/lib/auth"; +import { toNextJsHandler } from "better-auth/next-js"; + +export const { POST, GET } = toNextJsHandler(auth); +``` + +### Next.js (Pages Router) + +```ts +// pages/api/auth/[...all].ts +import { auth } from "@/lib/auth"; +import { toNextJsHandler } from "better-auth/next-js"; + +export default toNextJsHandler(auth); +``` + +### Nuxt + +```ts +// server/api/auth/[...all].ts +import { auth } from "~/utils/auth"; +import { toWebRequest } from "better-auth/utils/web"; + +export default defineEventHandler((event) => { + return auth.handler(toWebRequest(event)); +}); +``` + +### SvelteKit + +```ts +// hooks.server.ts +import { auth } from "$lib/auth"; +import { svelteKitHandler } from "better-auth/svelte-kit"; + +export async function handle({ event, resolve }) { + return svelteKitHandler({ event, resolve, auth }); +} +``` + +### Astro + +```ts +// pages/api/auth/[...all].ts +import { auth } from "@/lib/auth"; + +export async function ALL({ request }: { request: Request }) { + return auth.handler(request); +} +``` + +### Hono + +```ts +import { Hono } from "hono"; +import { auth } from "./auth"; + +const app = new Hono(); + +app.on(["POST", "GET"], "/api/auth/*", (c) => { + return auth.handler(c.req.raw); +}); +``` + +### Express + +```ts +import express from "express"; +import { toNodeHandler } from "better-auth/node"; +import { auth } from "./auth"; + +const app = express(); + +app.all("/api/auth/*", toNodeHandler(auth)); +``` + +## Protected Routes + +### Next.js Middleware + +```ts +// middleware.ts +import { auth } from "@/lib/auth"; +import { NextRequest, NextResponse } from "next/server"; + +export async function middleware(request: NextRequest) { + const session = await auth.api.getSession({ + headers: request.headers + }); + + if (!session) { + return NextResponse.redirect(new URL("/login", request.url)); + } + + return NextResponse.next(); +} + +export const config = { + matcher: ["/dashboard/:path*", "/profile/:path*"] +}; +``` + +### SvelteKit Hooks + +```ts +// hooks.server.ts +import { auth } from "$lib/auth"; +import { redirect } from "@sveltejs/kit"; + +export async function handle({ event, resolve }) { + const session = await auth.api.getSession({ + headers: event.request.headers + }); + + if (event.url.pathname.startsWith("/dashboard") && !session) { + throw redirect(303, "/login"); + } + + return resolve(event); +} +``` + +### Nuxt Middleware + +```ts +// middleware/auth.ts +export default defineNuxtRouteMiddleware(async (to) => { + const { data: session } = await useAuthSession(); + + if (!session.value && to.path.startsWith("/dashboard")) { + return navigateTo("/login"); + } +}); +``` + +## User Profile Management + +### Get Current User + +```ts +const { data: session } = await authClient.getSession(); +console.log(session.user); +``` + +### Update User Profile + +```ts +await authClient.updateUser({ + name: "New Name", + image: "https://example.com/new-avatar.jpg", + // Custom fields if defined in schema +}); +``` + +### Delete User Account + +```ts +await authClient.deleteUser({ + password: "currentPassword", // Required for security + callbackURL: "/" // Redirect after deletion +}); +``` + +## Best Practices + +1. **Password Security**: Enforce strong password requirements +2. **Email Verification**: Enable for production to prevent spam +3. **Rate Limiting**: Prevent brute force attacks (see advanced-features.md) +4. **HTTPS**: Always use HTTPS in production +5. **Error Messages**: Don't reveal if email exists during login +6. **Session Security**: Use secure, httpOnly cookies +7. **CSRF Protection**: Better Auth handles this automatically +8. **Password Reset**: Set short expiration for reset tokens +9. **Account Lockout**: Consider implementing after N failed attempts +10. **Audit Logs**: Track auth events for security monitoring diff --git a/skills/better-auth/references/oauth-providers.md b/skills/better-auth/references/oauth-providers.md new file mode 100644 index 0000000..5a92bec --- /dev/null +++ b/skills/better-auth/references/oauth-providers.md @@ -0,0 +1,430 @@ +# OAuth Providers + +Better Auth provides built-in OAuth 2.0 support for social authentication. No plugins required. + +## Supported Providers + +GitHub, Google, Apple, Discord, Facebook, Microsoft, Twitter/X, Spotify, Twitch, LinkedIn, Dropbox, GitLab, and more. + +## Basic OAuth Setup + +### Server Configuration + +```ts +import { betterAuth } from "better-auth"; + +export const auth = betterAuth({ + socialProviders: { + github: { + clientId: process.env.GITHUB_CLIENT_ID!, + clientSecret: process.env.GITHUB_CLIENT_SECRET!, + // Optional: custom scopes + scope: ["user:email", "read:user"] + }, + google: { + clientId: process.env.GOOGLE_CLIENT_ID!, + clientSecret: process.env.GOOGLE_CLIENT_SECRET!, + scope: ["openid", "email", "profile"] + }, + discord: { + clientId: process.env.DISCORD_CLIENT_ID!, + clientSecret: process.env.DISCORD_CLIENT_SECRET!, + } + } +}); +``` + +### Client Usage + +```ts +import { authClient } from "@/lib/auth-client"; + +// Basic sign in +await authClient.signIn.social({ + provider: "github", + callbackURL: "/dashboard" +}); + +// With callbacks +await authClient.signIn.social({ + provider: "google", + callbackURL: "/dashboard", + errorCallbackURL: "/error", + newUserCallbackURL: "/welcome", // For first-time users +}); +``` + +## Provider Configuration + +### GitHub OAuth + +1. Create OAuth App at https://github.com/settings/developers +2. Set Authorization callback URL: `http://localhost:3000/api/auth/callback/github` +3. Add credentials to `.env`: + +```env +GITHUB_CLIENT_ID=your_client_id +GITHUB_CLIENT_SECRET=your_client_secret +``` + +### Google OAuth + +1. Create project at https://console.cloud.google.com +2. Enable Google+ API +3. Create OAuth 2.0 credentials +4. Add authorized redirect URI: `http://localhost:3000/api/auth/callback/google` +5. Add credentials to `.env`: + +```env +GOOGLE_CLIENT_ID=your_client_id.apps.googleusercontent.com +GOOGLE_CLIENT_SECRET=your_client_secret +``` + +### Discord OAuth + +1. Create application at https://discord.com/developers/applications +2. Add OAuth2 redirect: `http://localhost:3000/api/auth/callback/discord` +3. Add credentials: + +```env +DISCORD_CLIENT_ID=your_client_id +DISCORD_CLIENT_SECRET=your_client_secret +``` + +### Apple Sign In + +```ts +export const auth = betterAuth({ + socialProviders: { + apple: { + clientId: process.env.APPLE_CLIENT_ID!, + clientSecret: process.env.APPLE_CLIENT_SECRET!, + teamId: process.env.APPLE_TEAM_ID!, + keyId: process.env.APPLE_KEY_ID!, + privateKey: process.env.APPLE_PRIVATE_KEY! + } + } +}); +``` + +### Microsoft/Azure AD + +```ts +export const auth = betterAuth({ + socialProviders: { + microsoft: { + clientId: process.env.MICROSOFT_CLIENT_ID!, + clientSecret: process.env.MICROSOFT_CLIENT_SECRET!, + tenantId: process.env.MICROSOFT_TENANT_ID, // Optional: for specific tenant + } + } +}); +``` + +### Twitter/X OAuth + +```ts +export const auth = betterAuth({ + socialProviders: { + twitter: { + clientId: process.env.TWITTER_CLIENT_ID!, + clientSecret: process.env.TWITTER_CLIENT_SECRET!, + } + } +}); +``` + +## Custom OAuth Provider + +Add custom OAuth 2.0 provider: + +```ts +import { betterAuth } from "better-auth"; + +export const auth = betterAuth({ + socialProviders: { + customProvider: { + clientId: process.env.CUSTOM_CLIENT_ID!, + clientSecret: process.env.CUSTOM_CLIENT_SECRET!, + authorizationUrl: "https://provider.com/oauth/authorize", + tokenUrl: "https://provider.com/oauth/token", + userInfoUrl: "https://provider.com/oauth/userinfo", + scope: ["email", "profile"], + // Map provider user data to Better Auth user + mapProfile: (profile) => ({ + id: profile.id, + email: profile.email, + name: profile.name, + image: profile.avatar_url + }) + } + } +}); +``` + +## Account Linking + +Link multiple OAuth providers to same user account. + +### Server Setup + +```ts +export const auth = betterAuth({ + account: { + accountLinking: { + enabled: true, + trustedProviders: ["google", "github"] // Auto-link these providers + } + } +}); +``` + +### Client Usage + +```ts +// Link new provider to existing account +await authClient.linkSocial({ + provider: "google", + callbackURL: "/profile" +}); + +// List linked accounts +const { data: session } = await authClient.getSession(); +const accounts = session.user.accounts; + +// Unlink account +await authClient.unlinkAccount({ + accountId: "account-id" +}); +``` + +## Token Management + +### Access OAuth Tokens + +```ts +// Server-side +const session = await auth.api.getSession({ + headers: request.headers +}); + +const accounts = await auth.api.listAccounts({ + userId: session.user.id +}); + +// Get specific provider token +const githubAccount = accounts.find(a => a.providerId === "github"); +const accessToken = githubAccount.accessToken; +const refreshToken = githubAccount.refreshToken; +``` + +### Refresh Tokens + +```ts +// Manually refresh OAuth token +const newToken = await auth.api.refreshToken({ + accountId: "account-id" +}); +``` + +### Use Provider API + +```ts +// Example: Use GitHub token to fetch repos +const githubAccount = accounts.find(a => a.providerId === "github"); + +const response = await fetch("https://api.github.com/user/repos", { + headers: { + Authorization: `Bearer ${githubAccount.accessToken}` + } +}); + +const repos = await response.json(); +``` + +## Advanced OAuth Configuration + +### Custom Scopes + +```ts +export const auth = betterAuth({ + socialProviders: { + github: { + clientId: process.env.GITHUB_CLIENT_ID!, + clientSecret: process.env.GITHUB_CLIENT_SECRET!, + scope: [ + "user:email", + "read:user", + "repo", // Access repositories + "gist" // Access gists + ] + } + } +}); +``` + +### State Parameter + +Better Auth automatically handles OAuth state parameter for CSRF protection. + +```ts +// Custom state validation +export const auth = betterAuth({ + advanced: { + generateState: async () => { + // Custom state generation + return crypto.randomUUID(); + }, + validateState: async (state: string) => { + // Custom state validation + return true; + } + } +}); +``` + +### PKCE Support + +Better Auth automatically uses PKCE (Proof Key for Code Exchange) for supported providers. + +```ts +export const auth = betterAuth({ + socialProviders: { + customProvider: { + pkce: true, // Enable PKCE + // ... other config + } + } +}); +``` + +## Error Handling + +### Client-Side + +```ts +await authClient.signIn.social({ + provider: "github", + errorCallbackURL: "/auth/error" +}, { + onError: (ctx) => { + console.error("OAuth error:", ctx.error); + // Handle specific errors + if (ctx.error.code === "OAUTH_ACCOUNT_ALREADY_LINKED") { + alert("This account is already linked to another user"); + } + } +}); +``` + +### Server-Side + +```ts +export const auth = betterAuth({ + callbacks: { + async onOAuthError({ error, provider }) { + console.error(`OAuth error with ${provider}:`, error); + // Log to monitoring service + await logError(error); + } + } +}); +``` + +## Callback URLs + +### Development + +``` +http://localhost:3000/api/auth/callback/{provider} +``` + +### Production + +``` +https://yourdomain.com/api/auth/callback/{provider} +``` + +**Important:** Add all callback URLs to OAuth provider settings. + +## UI Components + +### Sign In Button (React) + +```tsx +import { authClient } from "@/lib/auth-client"; + +export function SocialSignIn() { + const handleOAuth = async (provider: string) => { + await authClient.signIn.social({ + provider, + callbackURL: "/dashboard" + }); + }; + + return ( +
+ + + +
+ ); +} +``` + +## Best Practices + +1. **Callback URLs**: Add all environments (dev, staging, prod) to OAuth app +2. **Scopes**: Request minimum scopes needed +3. **Token Storage**: Better Auth stores tokens securely in database +4. **Token Refresh**: Implement automatic token refresh for long-lived sessions +5. **Account Linking**: Enable for better UX when user signs in with different providers +6. **Error Handling**: Provide clear error messages for OAuth failures +7. **Provider Icons**: Use official brand assets for OAuth buttons +8. **Mobile Deep Links**: Configure deep links for mobile OAuth flows +9. **Email Matching**: Consider auto-linking accounts with same email +10. **Privacy**: Inform users what data you access from OAuth providers + +## Common Issues + +### Redirect URI Mismatch + +Ensure callback URL in OAuth app matches exactly: +``` +http://localhost:3000/api/auth/callback/github +``` + +### Missing Scopes + +Add required scopes for email access: +```ts +scope: ["user:email"] // GitHub +scope: ["email"] // Google +``` + +### HTTPS Required + +Some providers (Apple, Microsoft) require HTTPS callbacks. Use ngrok for local development: +```bash +ngrok http 3000 +``` + +### CORS Errors + +Configure CORS if frontend/backend on different domains: +```ts +export const auth = betterAuth({ + advanced: { + corsOptions: { + origin: ["https://yourdomain.com"], + credentials: true + } + } +}); +``` diff --git a/skills/better-auth/scripts/better_auth_init.py b/skills/better-auth/scripts/better_auth_init.py new file mode 100644 index 0000000..3e51705 --- /dev/null +++ b/skills/better-auth/scripts/better_auth_init.py @@ -0,0 +1,521 @@ +#!/usr/bin/env python3 +""" +Better Auth Initialization Script + +Interactive script to initialize Better Auth configuration. +Supports multiple databases, ORMs, and authentication methods. + +.env loading order: process.env > skill/.env > skills/.env > .claude/.env +""" + +import os +import sys +import json +import secrets +from pathlib import Path +from typing import Optional, Dict, Any, List +from dataclasses import dataclass + + +@dataclass +class EnvConfig: + """Environment configuration holder.""" + secret: str + url: str + database_url: Optional[str] = None + github_client_id: Optional[str] = None + github_client_secret: Optional[str] = None + google_client_id: Optional[str] = None + google_client_secret: Optional[str] = None + + +class BetterAuthInit: + """Better Auth configuration initializer.""" + + def __init__(self, project_root: Optional[Path] = None): + """ + Initialize the Better Auth configuration tool. + + Args: + project_root: Project root directory. Auto-detected if not provided. + """ + self.project_root = project_root or self._find_project_root() + self.env_config: Optional[EnvConfig] = None + + @staticmethod + def _find_project_root() -> Path: + """ + Find project root by looking for package.json. + + Returns: + Path to project root. + + Raises: + RuntimeError: If project root cannot be found. + """ + current = Path.cwd() + while current != current.parent: + if (current / "package.json").exists(): + return current + current = current.parent + + raise RuntimeError("Could not find project root (no package.json found)") + + def _load_env_files(self) -> Dict[str, str]: + """ + Load environment variables from .env files in order. + + Loading order: process.env > skill/.env > skills/.env > .claude/.env + + Returns: + Dictionary of environment variables. + """ + env_vars = {} + + # Define search paths in reverse priority order + skill_dir = Path(__file__).parent.parent + env_paths = [ + self.project_root / ".claude" / ".env", + self.project_root / ".claude" / "skills" / ".env", + skill_dir / ".env", + ] + + # Load from files (lowest priority first) + for env_path in env_paths: + if env_path.exists(): + env_vars.update(self._parse_env_file(env_path)) + + # Override with process environment (highest priority) + env_vars.update(os.environ) + + return env_vars + + @staticmethod + def _parse_env_file(path: Path) -> Dict[str, str]: + """ + Parse .env file into dictionary. + + Args: + path: Path to .env file. + + Returns: + Dictionary of key-value pairs. + """ + env_vars = {} + try: + with open(path, "r") as f: + for line in f: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + key, value = line.split("=", 1) + # Remove quotes if present + value = value.strip().strip('"').strip("'") + env_vars[key.strip()] = value + except Exception as e: + print(f"Warning: Could not parse {path}: {e}") + + return env_vars + + @staticmethod + def generate_secret(length: int = 32) -> str: + """ + Generate cryptographically secure random secret. + + Args: + length: Length of secret in bytes. + + Returns: + Hex-encoded secret string. + """ + return secrets.token_hex(length) + + def prompt_database(self) -> Dict[str, Any]: + """ + Prompt user for database configuration. + + Returns: + Database configuration dictionary. + """ + print("\nDatabase Configuration") + print("=" * 50) + print("1. Direct Connection (PostgreSQL/MySQL/SQLite)") + print("2. Drizzle ORM") + print("3. Prisma") + print("4. Kysely") + print("5. MongoDB") + + choice = input("\nSelect database option (1-5): ").strip() + + db_configs = { + "1": self._prompt_direct_db, + "2": self._prompt_drizzle, + "3": self._prompt_prisma, + "4": self._prompt_kysely, + "5": self._prompt_mongodb, + } + + handler = db_configs.get(choice) + if not handler: + print("Invalid choice. Defaulting to direct PostgreSQL.") + return self._prompt_direct_db() + + return handler() + + def _prompt_direct_db(self) -> Dict[str, Any]: + """Prompt for direct database connection.""" + print("\nDatabase Type:") + print("1. PostgreSQL") + print("2. MySQL") + print("3. SQLite") + + db_type = input("Select (1-3): ").strip() + + if db_type == "3": + db_path = input("SQLite file path [./dev.db]: ").strip() or "./dev.db" + return { + "type": "sqlite", + "import": "import Database from 'better-sqlite3';", + "config": f'database: new Database("{db_path}")' + } + elif db_type == "2": + db_url = input("MySQL connection string: ").strip() + return { + "type": "mysql", + "import": "import { createPool } from 'mysql2/promise';", + "config": f"database: createPool({{ connectionString: process.env.DATABASE_URL }})", + "env_var": ("DATABASE_URL", db_url) + } + else: + db_url = input("PostgreSQL connection string: ").strip() + return { + "type": "postgresql", + "import": "import { Pool } from 'pg';", + "config": "database: new Pool({ connectionString: process.env.DATABASE_URL })", + "env_var": ("DATABASE_URL", db_url) + } + + def _prompt_drizzle(self) -> Dict[str, Any]: + """Prompt for Drizzle ORM configuration.""" + print("\nDrizzle Provider:") + print("1. PostgreSQL") + print("2. MySQL") + print("3. SQLite") + + provider = input("Select (1-3): ").strip() + provider_map = {"1": "pg", "2": "mysql", "3": "sqlite"} + provider_name = provider_map.get(provider, "pg") + + return { + "type": "drizzle", + "provider": provider_name, + "import": "import { drizzleAdapter } from 'better-auth/adapters/drizzle';\nimport { db } from '@/db';", + "config": f"database: drizzleAdapter(db, {{ provider: '{provider_name}' }})" + } + + def _prompt_prisma(self) -> Dict[str, Any]: + """Prompt for Prisma configuration.""" + print("\nPrisma Provider:") + print("1. PostgreSQL") + print("2. MySQL") + print("3. SQLite") + + provider = input("Select (1-3): ").strip() + provider_map = {"1": "postgresql", "2": "mysql", "3": "sqlite"} + provider_name = provider_map.get(provider, "postgresql") + + return { + "type": "prisma", + "provider": provider_name, + "import": "import { prismaAdapter } from 'better-auth/adapters/prisma';\nimport { PrismaClient } from '@prisma/client';\n\nconst prisma = new PrismaClient();", + "config": f"database: prismaAdapter(prisma, {{ provider: '{provider_name}' }})" + } + + def _prompt_kysely(self) -> Dict[str, Any]: + """Prompt for Kysely configuration.""" + return { + "type": "kysely", + "import": "import { kyselyAdapter } from 'better-auth/adapters/kysely';\nimport { db } from '@/db';", + "config": "database: kyselyAdapter(db, { provider: 'pg' })" + } + + def _prompt_mongodb(self) -> Dict[str, Any]: + """Prompt for MongoDB configuration.""" + mongo_uri = input("MongoDB connection string: ").strip() + db_name = input("Database name: ").strip() + + return { + "type": "mongodb", + "import": "import { mongodbAdapter } from 'better-auth/adapters/mongodb';\nimport { client } from '@/db';", + "config": f"database: mongodbAdapter(client, {{ databaseName: '{db_name}' }})", + "env_var": ("MONGODB_URI", mongo_uri) + } + + def prompt_auth_methods(self) -> List[str]: + """ + Prompt user for authentication methods. + + Returns: + List of selected auth method codes. + """ + print("\nAuthentication Methods") + print("=" * 50) + print("Select authentication methods (space-separated, e.g., '1 2 3'):") + print("1. Email/Password") + print("2. GitHub OAuth") + print("3. Google OAuth") + print("4. Discord OAuth") + print("5. Two-Factor Authentication (2FA)") + print("6. Passkeys (WebAuthn)") + print("7. Magic Link") + print("8. Username") + + choices = input("\nYour selection: ").strip().split() + return [c for c in choices if c in "12345678"] + + def generate_auth_config( + self, + db_config: Dict[str, Any], + auth_methods: List[str], + ) -> str: + """ + Generate auth.ts configuration file content. + + Args: + db_config: Database configuration. + auth_methods: Selected authentication methods. + + Returns: + Generated TypeScript configuration code. + """ + imports = ["import { betterAuth } from 'better-auth';"] + plugins = [] + plugin_imports = [] + config_parts = [] + + # Database import + if db_config.get("import"): + imports.append(db_config["import"]) + + # Email/Password + if "1" in auth_methods: + config_parts.append(""" emailAndPassword: { + enabled: true, + autoSignIn: true + }""") + + # OAuth providers + social_providers = [] + if "2" in auth_methods: + social_providers.append(""" github: { + clientId: process.env.GITHUB_CLIENT_ID!, + clientSecret: process.env.GITHUB_CLIENT_SECRET!, + }""") + + if "3" in auth_methods: + social_providers.append(""" google: { + clientId: process.env.GOOGLE_CLIENT_ID!, + clientSecret: process.env.GOOGLE_CLIENT_SECRET!, + }""") + + if "4" in auth_methods: + social_providers.append(""" discord: { + clientId: process.env.DISCORD_CLIENT_ID!, + clientSecret: process.env.DISCORD_CLIENT_SECRET!, + }""") + + if social_providers: + config_parts.append(f" socialProviders: {{\n{',\\n'.join(social_providers)}\n }}") + + # Plugins + if "5" in auth_methods: + plugin_imports.append("import { twoFactor } from 'better-auth/plugins';") + plugins.append("twoFactor()") + + if "6" in auth_methods: + plugin_imports.append("import { passkey } from 'better-auth/plugins';") + plugins.append("passkey()") + + if "7" in auth_methods: + plugin_imports.append("import { magicLink } from 'better-auth/plugins';") + plugins.append("""magicLink({ + sendMagicLink: async ({ email, url }) => { + // TODO: Implement email sending + console.log(`Magic link for ${email}: ${url}`); + } + })""") + + if "8" in auth_methods: + plugin_imports.append("import { username } from 'better-auth/plugins';") + plugins.append("username()") + + # Combine all imports + all_imports = imports + plugin_imports + + # Build config + config_body = ",\n".join(config_parts) + + if plugins: + plugins_str = ",\n ".join(plugins) + config_body += f",\n plugins: [\n {plugins_str}\n ]" + + # Final output + return f"""{chr(10).join(all_imports)} + +export const auth = betterAuth({{ + {db_config["config"]}, +{config_body} +}}); +""" + + def generate_env_file( + self, + db_config: Dict[str, Any], + auth_methods: List[str] + ) -> str: + """ + Generate .env file content. + + Args: + db_config: Database configuration. + auth_methods: Selected authentication methods. + + Returns: + Generated .env file content. + """ + env_vars = [ + f"BETTER_AUTH_SECRET={self.generate_secret()}", + "BETTER_AUTH_URL=http://localhost:3000", + ] + + # Database URL + if db_config.get("env_var"): + key, value = db_config["env_var"] + env_vars.append(f"{key}={value}") + + # OAuth credentials + if "2" in auth_methods: + env_vars.extend([ + "GITHUB_CLIENT_ID=your_github_client_id", + "GITHUB_CLIENT_SECRET=your_github_client_secret", + ]) + + if "3" in auth_methods: + env_vars.extend([ + "GOOGLE_CLIENT_ID=your_google_client_id", + "GOOGLE_CLIENT_SECRET=your_google_client_secret", + ]) + + if "4" in auth_methods: + env_vars.extend([ + "DISCORD_CLIENT_ID=your_discord_client_id", + "DISCORD_CLIENT_SECRET=your_discord_client_secret", + ]) + + return "\n".join(env_vars) + "\n" + + def run(self) -> None: + """Run interactive initialization.""" + print("=" * 50) + print("Better Auth Configuration Generator") + print("=" * 50) + + # Load existing env + env_vars = self._load_env_files() + + # Prompt for configuration + db_config = self.prompt_database() + auth_methods = self.prompt_auth_methods() + + # Generate files + auth_config = self.generate_auth_config(db_config, auth_methods) + env_content = self.generate_env_file(db_config, auth_methods) + + # Display output + print("\n" + "=" * 50) + print("Generated Configuration") + print("=" * 50) + + print("\n--- auth.ts ---") + print(auth_config) + + print("\n--- .env ---") + print(env_content) + + # Offer to save + save = input("\nSave configuration files? (y/N): ").strip().lower() + if save == "y": + self._save_files(auth_config, env_content) + else: + print("Configuration not saved.") + + def _save_files(self, auth_config: str, env_content: str) -> None: + """ + Save generated configuration files. + + Args: + auth_config: auth.ts content. + env_content: .env content. + """ + # Save auth.ts + auth_locations = [ + self.project_root / "lib" / "auth.ts", + self.project_root / "src" / "lib" / "auth.ts", + self.project_root / "utils" / "auth.ts", + self.project_root / "auth.ts", + ] + + print("\nWhere to save auth.ts?") + for i, loc in enumerate(auth_locations, 1): + print(f"{i}. {loc}") + print("5. Custom path") + + choice = input("Select (1-5): ").strip() + if choice == "5": + custom_path = input("Enter path: ").strip() + auth_path = Path(custom_path) + else: + idx = int(choice) - 1 if choice.isdigit() else 0 + auth_path = auth_locations[idx] + + auth_path.parent.mkdir(parents=True, exist_ok=True) + auth_path.write_text(auth_config) + print(f"Saved: {auth_path}") + + # Save .env + env_path = self.project_root / ".env" + if env_path.exists(): + backup = self.project_root / ".env.backup" + env_path.rename(backup) + print(f"Backed up existing .env to {backup}") + + env_path.write_text(env_content) + print(f"Saved: {env_path}") + + print("\nNext steps:") + print("1. Run: npx @better-auth/cli generate") + print("2. Apply database migrations") + print("3. Mount API handler in your framework") + print("4. Create client instance") + + +def main() -> int: + """ + Main entry point. + + Returns: + Exit code (0 for success, 1 for error). + """ + try: + initializer = BetterAuthInit() + initializer.run() + return 0 + except KeyboardInterrupt: + print("\n\nOperation cancelled.") + return 1 + except Exception as e: + print(f"\nError: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/better-auth/scripts/requirements.txt b/skills/better-auth/scripts/requirements.txt new file mode 100644 index 0000000..bafbe07 --- /dev/null +++ b/skills/better-auth/scripts/requirements.txt @@ -0,0 +1,15 @@ +# Better Auth Skill Dependencies +# Python 3.10+ required + +# No Python package dependencies - uses only standard library + +# Testing dependencies (dev) +pytest>=8.0.0 +pytest-cov>=4.1.0 +pytest-mock>=3.12.0 + +# Note: This script generates Better Auth configuration +# The actual Better Auth library is installed via npm/pnpm/yarn: +# npm install better-auth +# pnpm add better-auth +# yarn add better-auth diff --git a/skills/better-auth/scripts/tests/test_better_auth_init.py b/skills/better-auth/scripts/tests/test_better_auth_init.py new file mode 100644 index 0000000..910d569 --- /dev/null +++ b/skills/better-auth/scripts/tests/test_better_auth_init.py @@ -0,0 +1,421 @@ +""" +Tests for better_auth_init.py + +Covers main functionality with mocked I/O and file operations. +Target: >80% coverage +""" + +import sys +import pytest +from pathlib import Path +from unittest.mock import Mock, patch, mock_open, MagicMock +from io import StringIO + +# Add parent directory to path +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from better_auth_init import BetterAuthInit, EnvConfig, main + + +@pytest.fixture +def mock_project_root(tmp_path): + """Create mock project root with package.json.""" + (tmp_path / "package.json").write_text("{}") + return tmp_path + + +@pytest.fixture +def auth_init(mock_project_root): + """Create BetterAuthInit instance with mock project root.""" + return BetterAuthInit(project_root=mock_project_root) + + +class TestBetterAuthInit: + """Test BetterAuthInit class.""" + + def test_init_with_project_root(self, mock_project_root): + """Test initialization with explicit project root.""" + init = BetterAuthInit(project_root=mock_project_root) + assert init.project_root == mock_project_root + assert init.env_config is None + + def test_find_project_root_success(self, mock_project_root, monkeypatch): + """Test finding project root successfully.""" + monkeypatch.chdir(mock_project_root) + init = BetterAuthInit() + assert init.project_root == mock_project_root + + def test_find_project_root_failure(self, tmp_path, monkeypatch): + """Test failure to find project root.""" + # Create path without package.json + no_package_dir = tmp_path / "no-package" + no_package_dir.mkdir() + monkeypatch.chdir(no_package_dir) + + # Mock parent to stop infinite loop + with patch.object(Path, "parent", new_callable=lambda: property(lambda self: self)): + with pytest.raises(RuntimeError, match="Could not find project root"): + BetterAuthInit() + + def test_generate_secret(self): + """Test secret generation.""" + secret = BetterAuthInit.generate_secret() + assert len(secret) == 64 # 32 bytes = 64 hex chars + assert all(c in "0123456789abcdef" for c in secret) + + # Test custom length + secret = BetterAuthInit.generate_secret(length=16) + assert len(secret) == 32 # 16 bytes = 32 hex chars + + def test_parse_env_file(self, tmp_path): + """Test parsing .env file.""" + env_content = """ +# Comment +KEY1=value1 +KEY2="value2" +KEY3='value3' +INVALID LINE +KEY4=value=with=equals +""" + env_file = tmp_path / ".env" + env_file.write_text(env_content) + + result = BetterAuthInit._parse_env_file(env_file) + + assert result["KEY1"] == "value1" + assert result["KEY2"] == "value2" + assert result["KEY3"] == "value3" + assert result["KEY4"] == "value=with=equals" + assert "INVALID" not in result + + def test_parse_env_file_missing(self, tmp_path): + """Test parsing missing .env file.""" + result = BetterAuthInit._parse_env_file(tmp_path / "nonexistent.env") + assert result == {} + + def test_load_env_files(self, auth_init, mock_project_root): + """Test loading environment variables from multiple files.""" + # Create .env files + claude_env = mock_project_root / ".claude" / ".env" + claude_env.parent.mkdir(parents=True, exist_ok=True) + claude_env.write_text("BASE_VAR=base\nOVERRIDE=claude") + + skills_env = mock_project_root / ".claude" / "skills" / ".env" + skills_env.parent.mkdir(parents=True, exist_ok=True) + skills_env.write_text("OVERRIDE=skills\nSKILLS_VAR=skills") + + # Mock process env (highest priority) + with patch.dict("os.environ", {"OVERRIDE": "process", "PROCESS_VAR": "process"}): + result = auth_init._load_env_files() + + assert result["BASE_VAR"] == "base" + assert result["SKILLS_VAR"] == "skills" + assert result["OVERRIDE"] == "process" # Process env wins + assert result["PROCESS_VAR"] == "process" + + def test_prompt_direct_db_sqlite(self, auth_init): + """Test prompting for SQLite database.""" + with patch("builtins.input", side_effect=["3", "./test.db"]): + config = auth_init._prompt_direct_db() + + assert config["type"] == "sqlite" + assert "better-sqlite3" in config["import"] + assert "./test.db" in config["config"] + + def test_prompt_direct_db_postgresql(self, auth_init): + """Test prompting for PostgreSQL database.""" + with patch("builtins.input", side_effect=["1", "postgresql://localhost/test"]): + config = auth_init._prompt_direct_db() + + assert config["type"] == "postgresql" + assert "pg" in config["import"] + assert config["env_var"] == ("DATABASE_URL", "postgresql://localhost/test") + + def test_prompt_direct_db_mysql(self, auth_init): + """Test prompting for MySQL database.""" + with patch("builtins.input", side_effect=["2", "mysql://localhost/test"]): + config = auth_init._prompt_direct_db() + + assert config["type"] == "mysql" + assert "mysql2" in config["import"] + assert config["env_var"][0] == "DATABASE_URL" + + def test_prompt_drizzle(self, auth_init): + """Test prompting for Drizzle ORM.""" + with patch("builtins.input", return_value="1"): + config = auth_init._prompt_drizzle() + + assert config["type"] == "drizzle" + assert config["provider"] == "pg" + assert "drizzleAdapter" in config["import"] + assert "drizzleAdapter" in config["config"] + + def test_prompt_prisma(self, auth_init): + """Test prompting for Prisma.""" + with patch("builtins.input", return_value="2"): + config = auth_init._prompt_prisma() + + assert config["type"] == "prisma" + assert config["provider"] == "mysql" + assert "prismaAdapter" in config["import"] + assert "PrismaClient" in config["import"] + + def test_prompt_kysely(self, auth_init): + """Test prompting for Kysely.""" + config = auth_init._prompt_kysely() + + assert config["type"] == "kysely" + assert "kyselyAdapter" in config["import"] + + def test_prompt_mongodb(self, auth_init): + """Test prompting for MongoDB.""" + with patch("builtins.input", side_effect=["mongodb://localhost/test", "mydb"]): + config = auth_init._prompt_mongodb() + + assert config["type"] == "mongodb" + assert "mongodbAdapter" in config["import"] + assert "mydb" in config["config"] + assert config["env_var"] == ("MONGODB_URI", "mongodb://localhost/test") + + def test_prompt_database(self, auth_init): + """Test database prompting with different choices.""" + # Test valid choice + with patch("builtins.input", side_effect=["3", "1"]): + config = auth_init.prompt_database() + assert config["type"] == "prisma" + + # Test invalid choice (defaults to direct DB) + with patch("builtins.input", side_effect=["99", "1", "postgresql://localhost/test"]): + with patch("builtins.print"): + config = auth_init.prompt_database() + assert config["type"] == "postgresql" + + def test_prompt_auth_methods(self, auth_init): + """Test prompting for authentication methods.""" + with patch("builtins.input", return_value="1 2 3 5 8"): + with patch("builtins.print"): + methods = auth_init.prompt_auth_methods() + + assert methods == ["1", "2", "3", "5", "8"] + + def test_prompt_auth_methods_invalid(self, auth_init): + """Test filtering invalid auth method choices.""" + with patch("builtins.input", return_value="1 99 abc 3"): + with patch("builtins.print"): + methods = auth_init.prompt_auth_methods() + + assert methods == ["1", "3"] + + def test_generate_auth_config_basic(self, auth_init): + """Test generating basic auth config.""" + db_config = { + "import": "import Database from 'better-sqlite3';", + "config": "database: new Database('./dev.db')" + } + auth_methods = ["1"] # Email/password only + + config = auth_init.generate_auth_config(db_config, auth_methods) + + assert "import { betterAuth }" in config + assert "emailAndPassword" in config + assert "enabled: true" in config + assert "better-sqlite3" in config + + def test_generate_auth_config_with_oauth(self, auth_init): + """Test generating config with OAuth providers.""" + db_config = { + "import": "import { Pool } from 'pg';", + "config": "database: new Pool()" + } + auth_methods = ["1", "2", "3", "4"] # Email + GitHub + Google + Discord + + config = auth_init.generate_auth_config(db_config, auth_methods) + + assert "socialProviders" in config + assert "github:" in config + assert "google:" in config + assert "discord:" in config + assert "GITHUB_CLIENT_ID" in config + assert "GOOGLE_CLIENT_ID" in config + assert "DISCORD_CLIENT_ID" in config + + def test_generate_auth_config_with_plugins(self, auth_init): + """Test generating config with plugins.""" + db_config = {"import": "", "config": "database: db"} + auth_methods = ["5", "6", "7", "8"] # 2FA, Passkey, Magic Link, Username + + config = auth_init.generate_auth_config(db_config, auth_methods) + + assert "plugins:" in config + assert "twoFactor" in config + assert "passkey" in config + assert "magicLink" in config + assert "username" in config + assert "from 'better-auth/plugins'" in config + + def test_generate_env_file_basic(self, auth_init): + """Test generating basic .env file.""" + db_config = {"type": "sqlite"} + auth_methods = ["1"] + + env_content = auth_init.generate_env_file(db_config, auth_methods) + + assert "BETTER_AUTH_SECRET=" in env_content + assert "BETTER_AUTH_URL=http://localhost:3000" in env_content + assert len(env_content.split("\n")) >= 2 + + def test_generate_env_file_with_database_url(self, auth_init): + """Test generating .env with database URL.""" + db_config = { + "env_var": ("DATABASE_URL", "postgresql://localhost/test") + } + auth_methods = [] + + env_content = auth_init.generate_env_file(db_config, auth_methods) + + assert "DATABASE_URL=postgresql://localhost/test" in env_content + + def test_generate_env_file_with_oauth(self, auth_init): + """Test generating .env with OAuth credentials.""" + db_config = {} + auth_methods = ["2", "3", "4"] # GitHub, Google, Discord + + env_content = auth_init.generate_env_file(db_config, auth_methods) + + assert "GITHUB_CLIENT_ID=" in env_content + assert "GITHUB_CLIENT_SECRET=" in env_content + assert "GOOGLE_CLIENT_ID=" in env_content + assert "GOOGLE_CLIENT_SECRET=" in env_content + assert "DISCORD_CLIENT_ID=" in env_content + assert "DISCORD_CLIENT_SECRET=" in env_content + + def test_save_files(self, auth_init, mock_project_root): + """Test saving configuration files.""" + auth_config = "// auth config" + env_content = "SECRET=test" + + with patch("builtins.input", side_effect=["1"]): + auth_init._save_files(auth_config, env_content) + + # Check auth.ts was saved + auth_path = mock_project_root / "lib" / "auth.ts" + assert auth_path.exists() + assert auth_path.read_text() == auth_config + + # Check .env was saved + env_path = mock_project_root / ".env" + assert env_path.exists() + assert env_path.read_text() == env_content + + def test_save_files_custom_path(self, auth_init, mock_project_root): + """Test saving with custom path.""" + auth_config = "// config" + env_content = "SECRET=test" + + custom_path = str(mock_project_root / "custom" / "auth.ts") + with patch("builtins.input", side_effect=["5", custom_path]): + auth_init._save_files(auth_config, env_content) + + assert Path(custom_path).exists() + + def test_save_files_backup_existing_env(self, auth_init, mock_project_root): + """Test backing up existing .env file.""" + # Create existing .env + env_path = mock_project_root / ".env" + env_path.write_text("OLD_SECRET=old") + + auth_config = "// config" + env_content = "NEW_SECRET=new" + + with patch("builtins.input", return_value="1"): + auth_init._save_files(auth_config, env_content) + + # Check backup was created + backup_path = mock_project_root / ".env.backup" + assert backup_path.exists() + assert backup_path.read_text() == "OLD_SECRET=old" + + # Check new .env + assert env_path.read_text() == "NEW_SECRET=new" + + def test_run_full_flow(self, auth_init, mock_project_root): + """Test complete run flow.""" + inputs = [ + "1", # Direct DB + "1", # PostgreSQL + "postgresql://localhost/test", + "1 2", # Email + GitHub + "n" # Don't save + ] + + with patch("builtins.input", side_effect=inputs): + with patch("builtins.print"): + auth_init.run() + + # Should complete without errors + # Files not saved because user chose 'n' + assert not (mock_project_root / "auth.ts").exists() + + def test_run_save_files(self, auth_init, mock_project_root): + """Test run flow with file saving.""" + inputs = [ + "1", # Direct DB + "3", # SQLite + "", # Default path + "1", # Email only + "y", # Save + "1" # Save location + ] + + with patch("builtins.input", side_effect=inputs): + with patch("builtins.print"): + auth_init.run() + + # Check files were created + assert (mock_project_root / "lib" / "auth.ts").exists() + assert (mock_project_root / ".env").exists() + + +class TestMainFunction: + """Test main entry point.""" + + def test_main_success(self, tmp_path, monkeypatch): + """Test successful main execution.""" + (tmp_path / "package.json").write_text("{}") + monkeypatch.chdir(tmp_path) + + inputs = ["1", "3", "", "1", "n"] + + with patch("builtins.input", side_effect=inputs): + with patch("builtins.print"): + exit_code = main() + + assert exit_code == 0 + + def test_main_keyboard_interrupt(self, tmp_path, monkeypatch): + """Test main with keyboard interrupt.""" + (tmp_path / "package.json").write_text("{}") + monkeypatch.chdir(tmp_path) + + with patch("builtins.input", side_effect=KeyboardInterrupt()): + with patch("builtins.print"): + exit_code = main() + + assert exit_code == 1 + + def test_main_error(self, tmp_path, monkeypatch): + """Test main with error.""" + # No package.json - should fail + no_package = tmp_path / "no-package" + no_package.mkdir() + monkeypatch.chdir(no_package) + + with patch.object(Path, "parent", new_callable=lambda: property(lambda self: self)): + with patch("sys.stderr", new_callable=StringIO): + exit_code = main() + + assert exit_code == 1 + + +if __name__ == "__main__": + pytest.main([__file__, "-v", "--cov=better_auth_init", "--cov-report=term-missing"]) diff --git a/skills/chrome-devtools/SKILL.md b/skills/chrome-devtools/SKILL.md new file mode 100644 index 0000000..e6daa59 --- /dev/null +++ b/skills/chrome-devtools/SKILL.md @@ -0,0 +1,360 @@ +--- +name: chrome-devtools +description: Browser automation, debugging, and performance analysis using Puppeteer CLI scripts. Use for automating browsers, taking screenshots, analyzing performance, monitoring network traffic, web scraping, form automation, and JavaScript debugging. +license: Apache-2.0 +--- + +# Chrome DevTools Agent Skill + +Browser automation via executable Puppeteer scripts. All scripts output JSON for easy parsing. + +## Quick Start + +**CRITICAL**: Always check `pwd` before running scripts. + +### Installation + +#### Step 1: Install System Dependencies (Linux/WSL only) + +On Linux/WSL, Chrome requires system libraries. Install them first: + +```bash +pwd # Should show current working directory +cd .claude/skills/chrome-devtools/scripts +./install-deps.sh # Auto-detects OS and installs required libs +``` + +Supports: Ubuntu, Debian, Fedora, RHEL, CentOS, Arch, Manjaro + +**macOS/Windows**: Skip this step (dependencies bundled with Chrome) + +#### Step 2: Install Node Dependencies + +```bash +npm install # Installs puppeteer, debug, yargs +``` + +#### Step 3: Install ImageMagick (Optional, Recommended) + +ImageMagick enables automatic screenshot compression to keep files under 5MB: + +**macOS:** +```bash +brew install imagemagick +``` + +**Ubuntu/Debian/WSL:** +```bash +sudo apt-get install imagemagick +``` + +**Verify:** +```bash +magick -version # or: convert -version +``` + +Without ImageMagick, screenshots >5MB will not be compressed (may fail to load in Gemini/Claude). + +### Test +```bash +node navigate.js --url https://example.com +# Output: {"success": true, "url": "https://example.com", "title": "Example Domain"} +``` + +## Available Scripts + +All scripts are in `.claude/skills/chrome-devtools/scripts/` + +**CRITICAL**: Always check `pwd` before running scripts. + +### Script Usage +- `./scripts/README.md` + +### Core Automation +- `navigate.js` - Navigate to URLs +- `screenshot.js` - Capture screenshots (full page or element) +- `click.js` - Click elements +- `fill.js` - Fill form fields +- `evaluate.js` - Execute JavaScript in page context + +### Analysis & Monitoring +- `snapshot.js` - Extract interactive elements with metadata +- `console.js` - Monitor console messages/errors +- `network.js` - Track HTTP requests/responses +- `performance.js` - Measure Core Web Vitals + record traces + +## Usage Patterns + +### Single Command +```bash +pwd # Should show current working directory +cd .claude/skills/chrome-devtools/scripts +node screenshot.js --url https://example.com --output ./docs/screenshots/page.png +``` +**Important**: Always save screenshots to `./docs/screenshots` directory. + +### Automatic Image Compression +Screenshots are **automatically compressed** if they exceed 5MB to ensure compatibility with Gemini API and Claude Code (which have 5MB limits). This uses ImageMagick internally: + +```bash +# Default: auto-compress if >5MB +node screenshot.js --url https://example.com --output page.png + +# Custom size threshold (e.g., 3MB) +node screenshot.js --url https://example.com --output page.png --max-size 3 + +# Disable compression +node screenshot.js --url https://example.com --output page.png --no-compress +``` + +**Compression behavior:** +- PNG: Resizes to 90% + quality 85 (or 75% + quality 70 if still too large) +- JPEG: Quality 80 + progressive encoding (or quality 60 if still too large) +- Other formats: Converted to JPEG with compression +- Requires ImageMagick installed (see imagemagick skill) + +**Output includes compression info:** +```json +{ + "success": true, + "output": "/path/to/page.png", + "compressed": true, + "originalSize": 8388608, + "size": 3145728, + "compressionRatio": "62.50%", + "url": "https://example.com" +} +``` + +### Chain Commands (reuse browser) +```bash +# Keep browser open with --close false +node navigate.js --url https://example.com/login --close false +node fill.js --selector "#email" --value "user@example.com" --close false +node fill.js --selector "#password" --value "secret" --close false +node click.js --selector "button[type=submit]" +``` + +### Parse JSON Output +```bash +# Extract specific fields with jq +node performance.js --url https://example.com | jq '.vitals.LCP' + +# Save to file +node network.js --url https://example.com --output /tmp/requests.json +``` + +## Execution Protocol + +### Working Directory Verification + +BEFORE executing any script: +1. Check current working directory with `pwd` +2. Verify in `.claude/skills/chrome-devtools/scripts/` directory +3. If wrong directory, `cd` to correct location +4. Use absolute paths for all output files + +Example: +```bash +pwd # Should show: .../chrome-devtools/scripts +# If wrong: +cd .claude/skills/chrome-devtools/scripts +``` + +### Output Validation + +AFTER screenshot/capture operations: +1. Verify file created with `ls -lh ` +2. Read screenshot using Read tool to confirm content +3. Check JSON output for success:true +4. Report file size and compression status + +Example: +```bash +node screenshot.js --url https://example.com --output ./docs/screenshots/page.png +ls -lh ./docs/screenshots/page.png # Verify file exists +# Then use Read tool to visually inspect +``` + +5. Restart working directory to the project root. + +### Error Recovery + +If script fails: +1. Check error message for selector issues +2. Use snapshot.js to discover correct selectors +3. Try XPath selector if CSS selector fails +4. Verify element is visible and interactive + +Example: +```bash +# CSS selector fails +node click.js --url https://example.com --selector ".btn-submit" +# Error: waiting for selector ".btn-submit" failed + +# Discover correct selector +node snapshot.js --url https://example.com | jq '.elements[] | select(.tagName=="BUTTON")' + +# Try XPath +node click.js --url https://example.com --selector "//button[contains(text(),'Submit')]" +``` + +### Common Mistakes + +❌ Wrong working directory → output files go to wrong location +❌ Skipping output validation → silent failures +❌ Using complex CSS selectors without testing → selector errors +❌ Not checking element visibility → timeout errors + +✅ Always verify `pwd` before running scripts +✅ Always validate output after screenshots +✅ Use snapshot.js to discover selectors +✅ Test selectors with simple commands first + +## Common Workflows + +### Web Scraping +```bash +node evaluate.js --url https://example.com --script " + Array.from(document.querySelectorAll('.item')).map(el => ({ + title: el.querySelector('h2')?.textContent, + link: el.querySelector('a')?.href + })) +" | jq '.result' +``` + +### Performance Testing +```bash +PERF=$(node performance.js --url https://example.com) +LCP=$(echo $PERF | jq '.vitals.LCP') +if (( $(echo "$LCP < 2500" | bc -l) )); then + echo "✓ LCP passed: ${LCP}ms" +else + echo "✗ LCP failed: ${LCP}ms" +fi +``` + +### Form Automation +```bash +node fill.js --url https://example.com --selector "#search" --value "query" --close false +node click.js --selector "button[type=submit]" +``` + +### Error Monitoring +```bash +node console.js --url https://example.com --types error,warn --duration 5000 | jq '.messageCount' +``` + +## Script Options + +All scripts support: +- `--headless false` - Show browser window +- `--close false` - Keep browser open for chaining +- `--timeout 30000` - Set timeout (milliseconds) +- `--wait-until networkidle2` - Wait strategy + +See `./scripts/README.md` for complete options. + +## Output Format + +All scripts output JSON to stdout: +```json +{ + "success": true, + "url": "https://example.com", + ... // script-specific data +} +``` + +Errors go to stderr: +```json +{ + "success": false, + "error": "Error message" +} +``` + +## Finding Elements + +Use `snapshot.js` to discover selectors: +```bash +node snapshot.js --url https://example.com | jq '.elements[] | {tagName, text, selector}' +``` + +## Troubleshooting + +### Common Errors + +**"Cannot find package 'puppeteer'"** +- Run: `npm install` in the scripts directory + +**"error while loading shared libraries: libnss3.so"** (Linux/WSL) +- Missing system dependencies +- Fix: Run `./install-deps.sh` in scripts directory +- Manual install: `sudo apt-get install -y libnss3 libnspr4 libasound2t64 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1` + +**"Failed to launch the browser process"** +- Check system dependencies installed (Linux/WSL) +- Verify Chrome downloaded: `ls ~/.cache/puppeteer` +- Try: `npm rebuild` then `npm install` + +**Chrome not found** +- Puppeteer auto-downloads Chrome during `npm install` +- If failed, manually trigger: `npx puppeteer browsers install chrome` + +### Script Issues + +**Element not found** +- Get snapshot first to find correct selector: `node snapshot.js --url ` + +**Script hangs** +- Increase timeout: `--timeout 60000` +- Change wait strategy: `--wait-until load` or `--wait-until domcontentloaded` + +**Blank screenshot** +- Wait for page load: `--wait-until networkidle2` +- Increase timeout: `--timeout 30000` + +**Permission denied on scripts** +- Make executable: `chmod +x *.sh` + +**Screenshot too large (>5MB)** +- Install ImageMagick for automatic compression +- Manually set lower threshold: `--max-size 3` +- Use JPEG format instead of PNG: `--format jpeg --quality 80` +- Capture specific element instead of full page: `--selector .main-content` + +**Compression not working** +- Verify ImageMagick installed: `magick -version` or `convert -version` +- Check file was actually compressed in output JSON: `"compressed": true` +- For very large pages, use `--selector` to capture only needed area + +## Reference Documentation + +Detailed guides available in `./references/`: +- [CDP Domains Reference](./references/cdp-domains.md) - 47 Chrome DevTools Protocol domains +- [Puppeteer Quick Reference](./references/puppeteer-reference.md) - Complete Puppeteer API patterns +- [Performance Analysis Guide](./references/performance-guide.md) - Core Web Vitals optimization + +## Advanced Usage + +### Custom Scripts +Create custom scripts using shared library: +```javascript +import { getBrowser, getPage, closeBrowser, outputJSON } from './lib/browser.js'; +// Your automation logic +``` + +### Direct CDP Access +```javascript +const client = await page.createCDPSession(); +await client.send('Emulation.setCPUThrottlingRate', { rate: 4 }); +``` + +See reference documentation for advanced patterns and complete API coverage. + +## External Resources + +- [Puppeteer Documentation](https://pptr.dev/) +- [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/) +- [Scripts README](./scripts/README.md) diff --git a/skills/chrome-devtools/references/cdp-domains.md b/skills/chrome-devtools/references/cdp-domains.md new file mode 100644 index 0000000..b4cf0f6 --- /dev/null +++ b/skills/chrome-devtools/references/cdp-domains.md @@ -0,0 +1,694 @@ +# Chrome DevTools Protocol (CDP) Domains Reference + +Complete reference of CDP domains and their capabilities for browser automation and debugging. + +## Overview + +CDP is organized into **47 domains**, each providing specific browser capabilities. Domains are grouped by functionality: + +- **Core** - Fundamental browser control +- **DOM & Styling** - Page structure and styling +- **Network & Fetch** - HTTP traffic management +- **Page & Navigation** - Page lifecycle control +- **Storage & Data** - Browser storage APIs +- **Performance & Profiling** - Metrics and analysis +- **Emulation & Simulation** - Device and network emulation +- **Worker & Service** - Background tasks +- **Developer Tools** - Debugging support + +--- + +## Core Domains + +### Runtime +**Purpose:** Execute JavaScript, manage objects, handle promises + +**Key Commands:** +- `Runtime.evaluate(expression)` - Execute JavaScript +- `Runtime.callFunctionOn(functionDeclaration, objectId)` - Call function on object +- `Runtime.getProperties(objectId)` - Get object properties +- `Runtime.awaitPromise(promiseObjectId)` - Wait for promise resolution + +**Key Events:** +- `Runtime.consoleAPICalled` - Console message logged +- `Runtime.exceptionThrown` - Uncaught exception + +**Use Cases:** +- Execute custom JavaScript +- Access page data +- Monitor console output +- Handle exceptions + +--- + +### Debugger +**Purpose:** JavaScript debugging, breakpoints, stack traces + +**Key Commands:** +- `Debugger.enable()` - Enable debugger +- `Debugger.setBreakpoint(location)` - Set breakpoint +- `Debugger.pause()` - Pause execution +- `Debugger.resume()` - Resume execution +- `Debugger.stepOver/stepInto/stepOut()` - Step through code + +**Key Events:** +- `Debugger.paused` - Execution paused +- `Debugger.resumed` - Execution resumed +- `Debugger.scriptParsed` - Script loaded + +**Use Cases:** +- Debug JavaScript errors +- Inspect call stacks +- Set conditional breakpoints +- Source map support + +--- + +### Console (Deprecated - Use Runtime/Log) +**Purpose:** Legacy console message access + +**Note:** Use `Runtime.consoleAPICalled` event instead for new implementations. + +--- + +## DOM & Styling Domains + +### DOM +**Purpose:** Access and manipulate DOM tree + +**Key Commands:** +- `DOM.getDocument()` - Get root document node +- `DOM.querySelector(nodeId, selector)` - Query selector +- `DOM.querySelectorAll(nodeId, selector)` - Query all +- `DOM.getAttributes(nodeId)` - Get element attributes +- `DOM.setOuterHTML(nodeId, outerHTML)` - Replace element +- `DOM.getBoxModel(nodeId)` - Get element layout box +- `DOM.focus(nodeId)` - Focus element + +**Key Events:** +- `DOM.documentUpdated` - Document changed +- `DOM.setChildNodes` - Child nodes updated + +**Use Cases:** +- Navigate DOM tree +- Query elements +- Modify DOM structure +- Get element positions + +--- + +### CSS +**Purpose:** Inspect and modify CSS styles + +**Key Commands:** +- `CSS.enable()` - Enable CSS domain +- `CSS.getComputedStyleForNode(nodeId)` - Get computed styles +- `CSS.getInlineStylesForNode(nodeId)` - Get inline styles +- `CSS.getMatchedStylesForNode(nodeId)` - Get matched CSS rules +- `CSS.setStyleTexts(edits)` - Modify styles + +**Key Events:** +- `CSS.styleSheetAdded` - Stylesheet added +- `CSS.styleSheetChanged` - Stylesheet modified + +**Use Cases:** +- Inspect element styles +- Debug CSS issues +- Modify styles dynamically +- Extract stylesheet data + +--- + +### Accessibility +**Purpose:** Access accessibility tree + +**Key Commands:** +- `Accessibility.enable()` - Enable accessibility +- `Accessibility.getFullAXTree()` - Get complete AX tree +- `Accessibility.getPartialAXTree(nodeId)` - Get node subtree +- `Accessibility.queryAXTree(nodeId, role, name)` - Query AX tree + +**Use Cases:** +- Accessibility testing +- Screen reader simulation +- ARIA attribute inspection +- AX tree analysis + +--- + +## Network & Fetch Domains + +### Network +**Purpose:** Monitor and control HTTP traffic + +**Key Commands:** +- `Network.enable()` - Enable network tracking +- `Network.setCacheDisabled(cacheDisabled)` - Disable cache +- `Network.setExtraHTTPHeaders(headers)` - Add custom headers +- `Network.getCookies(urls)` - Get cookies +- `Network.setCookie(name, value, domain)` - Set cookie +- `Network.getResponseBody(requestId)` - Get response body +- `Network.emulateNetworkConditions(offline, latency, downloadThroughput, uploadThroughput)` - Throttle network + +**Key Events:** +- `Network.requestWillBeSent` - Request starting +- `Network.responseReceived` - Response received +- `Network.loadingFinished` - Request completed +- `Network.loadingFailed` - Request failed + +**Use Cases:** +- Monitor API calls +- Intercept requests +- Analyze response data +- Simulate slow networks +- Manage cookies + +--- + +### Fetch +**Purpose:** Intercept and modify network requests + +**Key Commands:** +- `Fetch.enable(patterns)` - Enable request interception +- `Fetch.continueRequest(requestId, url, method, headers)` - Continue/modify request +- `Fetch.fulfillRequest(requestId, responseCode, headers, body)` - Mock response +- `Fetch.failRequest(requestId, errorReason)` - Fail request + +**Key Events:** +- `Fetch.requestPaused` - Request intercepted + +**Use Cases:** +- Mock API responses +- Block requests +- Modify request/response +- Test error scenarios + +--- + +## Page & Navigation Domains + +### Page +**Purpose:** Control page lifecycle and navigation + +**Key Commands:** +- `Page.enable()` - Enable page domain +- `Page.navigate(url)` - Navigate to URL +- `Page.reload(ignoreCache)` - Reload page +- `Page.goBack()/goForward()` - Navigate history +- `Page.captureScreenshot(format, quality)` - Take screenshot +- `Page.printToPDF(landscape, displayHeaderFooter)` - Generate PDF +- `Page.getLayoutMetrics()` - Get page dimensions +- `Page.createIsolatedWorld(frameId)` - Create isolated context +- `Page.handleJavaScriptDialog(accept, promptText)` - Handle alerts/confirms + +**Key Events:** +- `Page.loadEventFired` - Page loaded +- `Page.domContentEventFired` - DOM ready +- `Page.frameNavigated` - Frame navigated +- `Page.javascriptDialogOpening` - Alert/confirm shown + +**Use Cases:** +- Navigate pages +- Capture screenshots +- Generate PDFs +- Handle popups +- Monitor page lifecycle + +--- + +### Target +**Purpose:** Manage browser targets (tabs, workers, frames) + +**Key Commands:** +- `Target.getTargets()` - List all targets +- `Target.createTarget(url)` - Open new tab +- `Target.closeTarget(targetId)` - Close tab +- `Target.attachToTarget(targetId)` - Attach debugger +- `Target.detachFromTarget(sessionId)` - Detach debugger +- `Target.setDiscoverTargets(discover)` - Auto-discover targets + +**Key Events:** +- `Target.targetCreated` - New target created +- `Target.targetDestroyed` - Target closed +- `Target.targetInfoChanged` - Target updated + +**Use Cases:** +- Multi-tab automation +- Service worker debugging +- Frame inspection +- Extension debugging + +--- + +### Input +**Purpose:** Simulate user input + +**Key Commands:** +- `Input.dispatchKeyEvent(type, key, code)` - Keyboard input +- `Input.dispatchMouseEvent(type, x, y, button)` - Mouse input +- `Input.dispatchTouchEvent(type, touchPoints)` - Touch input +- `Input.synthesizePinchGesture(x, y, scaleFactor)` - Pinch gesture +- `Input.synthesizeScrollGesture(x, y, xDistance, yDistance)` - Scroll + +**Use Cases:** +- Simulate clicks +- Type text +- Drag and drop +- Touch gestures +- Scroll pages + +--- + +## Storage & Data Domains + +### Storage +**Purpose:** Manage browser storage + +**Key Commands:** +- `Storage.getCookies(browserContextId)` - Get cookies +- `Storage.setCookies(cookies)` - Set cookies +- `Storage.clearCookies(browserContextId)` - Clear cookies +- `Storage.clearDataForOrigin(origin, storageTypes)` - Clear storage +- `Storage.getUsageAndQuota(origin)` - Get storage usage + +**Storage Types:** +- appcache, cookies, file_systems, indexeddb, local_storage, shader_cache, websql, service_workers, cache_storage + +**Use Cases:** +- Cookie management +- Clear browser data +- Inspect storage usage +- Test quota limits + +--- + +### DOMStorage +**Purpose:** Access localStorage/sessionStorage + +**Key Commands:** +- `DOMStorage.enable()` - Enable storage tracking +- `DOMStorage.getDOMStorageItems(storageId)` - Get items +- `DOMStorage.setDOMStorageItem(storageId, key, value)` - Set item +- `DOMStorage.removeDOMStorageItem(storageId, key)` - Remove item + +**Key Events:** +- `DOMStorage.domStorageItemsCleared` - Storage cleared +- `DOMStorage.domStorageItemAdded/Updated/Removed` - Item changed + +--- + +### IndexedDB +**Purpose:** Query IndexedDB databases + +**Key Commands:** +- `IndexedDB.requestDatabaseNames(securityOrigin)` - List databases +- `IndexedDB.requestDatabase(securityOrigin, databaseName)` - Get DB structure +- `IndexedDB.requestData(securityOrigin, databaseName, objectStoreName)` - Query data + +**Use Cases:** +- Inspect IndexedDB data +- Debug database issues +- Extract stored data + +--- + +### CacheStorage +**Purpose:** Manage Cache API + +**Key Commands:** +- `CacheStorage.requestCacheNames(securityOrigin)` - List caches +- `CacheStorage.requestCachedResponses(cacheId, securityOrigin)` - List cached responses +- `CacheStorage.deleteCache(cacheId)` - Delete cache + +**Use Cases:** +- Service worker cache inspection +- Offline functionality testing + +--- + +## Performance & Profiling Domains + +### Performance +**Purpose:** Collect performance metrics + +**Key Commands:** +- `Performance.enable()` - Enable performance tracking +- `Performance.disable()` - Disable tracking +- `Performance.getMetrics()` - Get current metrics + +**Metrics:** +- Timestamp, Documents, Frames, JSEventListeners, Nodes, LayoutCount, RecalcStyleCount, LayoutDuration, RecalcStyleDuration, ScriptDuration, TaskDuration, JSHeapUsedSize, JSHeapTotalSize + +**Use Cases:** +- Monitor page metrics +- Track memory usage +- Measure render times + +--- + +### PerformanceTimeline +**Purpose:** Access Performance Timeline API + +**Key Commands:** +- `PerformanceTimeline.enable(eventTypes)` - Subscribe to events + +**Event Types:** +- mark, measure, navigation, resource, longtask, paint, layout-shift + +**Key Events:** +- `PerformanceTimeline.timelineEventAdded` - New performance entry + +--- + +### Tracing +**Purpose:** Record Chrome trace + +**Key Commands:** +- `Tracing.start(categories, options)` - Start recording +- `Tracing.end()` - Stop recording +- `Tracing.requestMemoryDump()` - Capture memory snapshot + +**Trace Categories:** +- blink, cc, devtools, gpu, loading, navigation, rendering, v8, disabled-by-default-* + +**Key Events:** +- `Tracing.dataCollected` - Trace chunk received +- `Tracing.tracingComplete` - Recording finished + +**Use Cases:** +- Deep performance analysis +- Frame rendering profiling +- CPU flame graphs +- Memory profiling + +--- + +### Profiler +**Purpose:** CPU profiling + +**Key Commands:** +- `Profiler.enable()` - Enable profiler +- `Profiler.start()` - Start CPU profiling +- `Profiler.stop()` - Stop and get profile + +**Use Cases:** +- Find CPU bottlenecks +- Optimize JavaScript +- Generate flame graphs + +--- + +### HeapProfiler (via Memory domain) +**Purpose:** Memory profiling + +**Key Commands:** +- `Memory.getDOMCounters()` - Get DOM object counts +- `Memory.prepareForLeakDetection()` - Prepare leak detection +- `Memory.forciblyPurgeJavaScriptMemory()` - Force GC +- `Memory.setPressureNotificationsSuppressed(suppressed)` - Control memory warnings +- `Memory.simulatePressureNotification(level)` - Simulate memory pressure + +**Use Cases:** +- Detect memory leaks +- Analyze heap snapshots +- Monitor object counts + +--- + +## Emulation & Simulation Domains + +### Emulation +**Purpose:** Emulate device conditions + +**Key Commands:** +- `Emulation.setDeviceMetricsOverride(width, height, deviceScaleFactor, mobile)` - Emulate device +- `Emulation.setGeolocationOverride(latitude, longitude, accuracy)` - Fake location +- `Emulation.setEmulatedMedia(media, features)` - Emulate media type +- `Emulation.setTimezoneOverride(timezoneId)` - Override timezone +- `Emulation.setLocaleOverride(locale)` - Override language +- `Emulation.setUserAgentOverride(userAgent)` - Change user agent + +**Use Cases:** +- Mobile device testing +- Geolocation testing +- Print media emulation +- Timezone/locale testing + +--- + +### DeviceOrientation +**Purpose:** Simulate device orientation + +**Key Commands:** +- `DeviceOrientation.setDeviceOrientationOverride(alpha, beta, gamma)` - Set orientation + +**Use Cases:** +- Test accelerometer features +- Orientation-dependent layouts + +--- + +## Worker & Service Domains + +### ServiceWorker +**Purpose:** Manage service workers + +**Key Commands:** +- `ServiceWorker.enable()` - Enable tracking +- `ServiceWorker.unregister(scopeURL)` - Unregister worker +- `ServiceWorker.startWorker(scopeURL)` - Start worker +- `ServiceWorker.stopWorker(versionId)` - Stop worker +- `ServiceWorker.inspectWorker(versionId)` - Debug worker + +**Key Events:** +- `ServiceWorker.workerRegistrationUpdated` - Registration changed +- `ServiceWorker.workerVersionUpdated` - Version updated + +--- + +### WebAuthn +**Purpose:** Simulate WebAuthn/FIDO2 + +**Key Commands:** +- `WebAuthn.enable()` - Enable virtual authenticators +- `WebAuthn.addVirtualAuthenticator(options)` - Add virtual device +- `WebAuthn.removeVirtualAuthenticator(authenticatorId)` - Remove device +- `WebAuthn.addCredential(authenticatorId, credential)` - Add credential + +**Use Cases:** +- Test WebAuthn flows +- Simulate biometric auth +- Test security keys + +--- + +## Developer Tools Support + +### Inspector +**Purpose:** Protocol-level debugging + +**Key Events:** +- `Inspector.detached` - Debugger disconnected +- `Inspector.targetCrashed` - Target crashed + +--- + +### Log +**Purpose:** Collect browser logs + +**Key Commands:** +- `Log.enable()` - Enable log collection +- `Log.clear()` - Clear logs + +**Key Events:** +- `Log.entryAdded` - New log entry + +**Use Cases:** +- Collect console logs +- Monitor violations +- Track deprecations + +--- + +### DOMDebugger +**Purpose:** DOM-level debugging + +**Key Commands:** +- `DOMDebugger.setDOMBreakpoint(nodeId, type)` - Break on DOM changes +- `DOMDebugger.setEventListenerBreakpoint(eventName)` - Break on event +- `DOMDebugger.setXHRBreakpoint(url)` - Break on XHR + +**Breakpoint Types:** +- subtree-modified, attribute-modified, node-removed + +--- + +### DOMSnapshot +**Purpose:** Capture complete DOM snapshot + +**Key Commands:** +- `DOMSnapshot.captureSnapshot(computedStyles)` - Capture full DOM + +**Use Cases:** +- Export page structure +- Offline analysis +- DOM diffing + +--- + +### Audits (Lighthouse Integration) +**Purpose:** Run automated audits + +**Key Commands:** +- `Audits.enable()` - Enable audits +- `Audits.getEncodingIssues()` - Check encoding issues + +--- + +### LayerTree +**Purpose:** Inspect rendering layers + +**Key Commands:** +- `LayerTree.enable()` - Enable layer tracking +- `LayerTree.compositingReasons(layerId)` - Get why layer created + +**Key Events:** +- `LayerTree.layerTreeDidChange` - Layers changed + +**Use Cases:** +- Debug rendering performance +- Identify layer creation +- Optimize compositing + +--- + +## Other Domains + +### Browser +**Purpose:** Browser-level control + +**Key Commands:** +- `Browser.getVersion()` - Get browser info +- `Browser.getBrowserCommandLine()` - Get launch args +- `Browser.setPermission(permission, setting, origin)` - Set permissions +- `Browser.grantPermissions(permissions, origin)` - Grant permissions + +**Permissions:** +- geolocation, midi, notifications, push, camera, microphone, background-sync, sensors, accessibility-events, clipboard-read, clipboard-write, payment-handler + +--- + +### IO +**Purpose:** File I/O operations + +**Key Commands:** +- `IO.read(handle, offset, size)` - Read stream +- `IO.close(handle)` - Close stream + +**Use Cases:** +- Read large response bodies +- Process binary data + +--- + +### Media +**Purpose:** Inspect media players + +**Key Commands:** +- `Media.enable()` - Track media players + +**Key Events:** +- `Media.playerPropertiesChanged` - Player state changed +- `Media.playerEventsAdded` - Player events + +--- + +### BackgroundService +**Purpose:** Track background services + +**Key Commands:** +- `BackgroundService.startObserving(service)` - Track service + +**Services:** +- backgroundFetch, backgroundSync, pushMessaging, notifications, paymentHandler, periodicBackgroundSync + +--- + +## Domain Dependencies + +Some domains depend on others and must be enabled in order: + +``` +Runtime (no dependencies) + ↓ +DOM (depends on Runtime) + ↓ +CSS (depends on DOM) + +Network (no dependencies) + +Page (depends on Runtime) + ↓ +Target (depends on Page) + +Debugger (depends on Runtime) +``` + +## Quick Command Reference + +### Most Common Commands + +```javascript +// Navigation +Page.navigate(url) +Page.reload() + +// JavaScript Execution +Runtime.evaluate(expression) + +// DOM Access +DOM.getDocument() +DOM.querySelector(nodeId, selector) + +// Screenshots +Page.captureScreenshot(format, quality) + +// Network Monitoring +Network.enable() +// Listen for Network.requestWillBeSent events + +// Console Messages +// Listen for Runtime.consoleAPICalled events + +// Cookies +Network.getCookies(urls) +Network.setCookie(...) + +// Device Emulation +Emulation.setDeviceMetricsOverride(width, height, ...) + +// Performance +Performance.getMetrics() +Tracing.start(categories) +Tracing.end() +``` + +--- + +## Best Practices + +1. **Enable domains before use:** Always call `.enable()` for stateful domains +2. **Handle events:** Subscribe to events for real-time updates +3. **Clean up:** Disable domains when done to reduce overhead +4. **Use sessions:** Attach to specific targets for isolated debugging +5. **Handle errors:** Implement proper error handling for command failures +6. **Version awareness:** Check browser version for experimental API support + +--- + +## Additional Resources + +- [Protocol Viewer](https://chromedevtools.github.io/devtools-protocol/) - Interactive domain browser +- [Protocol JSON](https://chromedevtools.github.io/devtools-protocol/tot/json) - Machine-readable specification +- [Getting Started with CDP](https://github.com/aslushnikov/getting-started-with-cdp) +- [devtools-protocol NPM](https://www.npmjs.com/package/devtools-protocol) - TypeScript definitions diff --git a/skills/chrome-devtools/references/performance-guide.md b/skills/chrome-devtools/references/performance-guide.md new file mode 100644 index 0000000..206bb42 --- /dev/null +++ b/skills/chrome-devtools/references/performance-guide.md @@ -0,0 +1,940 @@ +# Performance Analysis Guide + +Comprehensive guide to analyzing web performance using Chrome DevTools Protocol, Puppeteer, and chrome-devtools skill. + +## Table of Contents + +- [Core Web Vitals](#core-web-vitals) +- [Performance Tracing](#performance-tracing) +- [Network Analysis](#network-analysis) +- [JavaScript Performance](#javascript-performance) +- [Rendering Performance](#rendering-performance) +- [Memory Analysis](#memory-analysis) +- [Optimization Strategies](#optimization-strategies) + +--- + +## Core Web Vitals + +### Overview + +Core Web Vitals are Google's standardized metrics for measuring user experience: + +- **LCP (Largest Contentful Paint)** - Loading performance (< 2.5s good) +- **FID (First Input Delay)** - Interactivity (< 100ms good) +- **CLS (Cumulative Layout Shift)** - Visual stability (< 0.1 good) + +### Measuring with chrome-devtools-mcp + +```javascript +// Start performance trace +await useTool('performance_start_trace', { + categories: ['loading', 'rendering', 'scripting'] +}); + +// Navigate to page +await useTool('navigate_page', { + url: 'https://example.com' +}); + +// Wait for complete load +await useTool('wait_for', { + waitUntil: 'networkidle' +}); + +// Stop trace and get data +await useTool('performance_stop_trace'); + +// Get AI-powered insights +const insights = await useTool('performance_analyze_insight'); + +// insights will include: +// - LCP timing +// - FID analysis +// - CLS score +// - Performance recommendations +``` + +### Measuring with Puppeteer + +```javascript +import puppeteer from 'puppeteer'; + +const browser = await puppeteer.launch(); +const page = await browser.newPage(); + +// Measure Core Web Vitals +await page.goto('https://example.com', { + waitUntil: 'networkidle2' +}); + +const vitals = await page.evaluate(() => { + return new Promise((resolve) => { + const vitals = { + LCP: null, + FID: null, + CLS: 0 + }; + + // LCP + new PerformanceObserver((list) => { + const entries = list.getEntries(); + vitals.LCP = entries[entries.length - 1].renderTime || + entries[entries.length - 1].loadTime; + }).observe({ entryTypes: ['largest-contentful-paint'] }); + + // FID + new PerformanceObserver((list) => { + vitals.FID = list.getEntries()[0].processingStart - + list.getEntries()[0].startTime; + }).observe({ entryTypes: ['first-input'] }); + + // CLS + new PerformanceObserver((list) => { + list.getEntries().forEach((entry) => { + if (!entry.hadRecentInput) { + vitals.CLS += entry.value; + } + }); + }).observe({ entryTypes: ['layout-shift'] }); + + // Wait 5 seconds for metrics + setTimeout(() => resolve(vitals), 5000); + }); +}); + +console.log('Core Web Vitals:', vitals); +``` + +### Other Important Metrics + +**TTFB (Time to First Byte)** +```javascript +const ttfb = await page.evaluate(() => { + const [navigationEntry] = performance.getEntriesByType('navigation'); + return navigationEntry.responseStart - navigationEntry.requestStart; +}); +``` + +**FCP (First Contentful Paint)** +```javascript +const fcp = await page.evaluate(() => { + const paintEntries = performance.getEntriesByType('paint'); + const fcpEntry = paintEntries.find(e => e.name === 'first-contentful-paint'); + return fcpEntry ? fcpEntry.startTime : null; +}); +``` + +**TTI (Time to Interactive)** +```javascript +// Requires lighthouse or manual calculation +const tti = await page.evaluate(() => { + // Complex calculation based on network idle and long tasks + // Best to use Lighthouse for accurate TTI +}); +``` + +--- + +## Performance Tracing + +### Chrome Trace Categories + +**Loading:** +- Page load events +- Resource loading +- Parser activity + +**Rendering:** +- Layout calculations +- Paint operations +- Compositing + +**Scripting:** +- JavaScript execution +- V8 compilation +- Garbage collection + +**Network:** +- HTTP requests +- WebSocket traffic +- Resource fetching + +**Input:** +- User input processing +- Touch/scroll events + +**GPU:** +- GPU operations +- Compositing work + +### Record Performance Trace + +**Using chrome-devtools-mcp:** +```javascript +// Start trace with specific categories +await useTool('performance_start_trace', { + categories: ['loading', 'rendering', 'scripting', 'network'] +}); + +// Perform actions +await useTool('navigate_page', { url: 'https://example.com' }); +await useTool('wait_for', { waitUntil: 'networkidle' }); + +// Optional: Interact with page +await useTool('click', { uid: 'button-uid' }); + +// Stop trace +const traceData = await useTool('performance_stop_trace'); + +// Analyze trace +const insights = await useTool('performance_analyze_insight'); +``` + +**Using Puppeteer:** +```javascript +// Start tracing +await page.tracing.start({ + path: 'trace.json', + categories: [ + 'devtools.timeline', + 'disabled-by-default-devtools.timeline', + 'disabled-by-default-v8.cpu_profiler' + ] +}); + +// Navigate +await page.goto('https://example.com', { + waitUntil: 'networkidle2' +}); + +// Stop tracing +await page.tracing.stop(); + +// Analyze in Chrome DevTools (chrome://tracing) +``` + +### Analyze Trace Data + +**Key Metrics from Trace:** + +1. **Main Thread Activity** + - JavaScript execution time + - Layout/reflow time + - Paint time + - Long tasks (> 50ms) + +2. **Network Waterfall** + - Request start times + - DNS lookup + - Connection time + - Download time + +3. **Rendering Pipeline** + - DOM construction + - Style calculation + - Layout + - Paint + - Composite + +**Common Issues to Look For:** +- Long tasks blocking main thread +- Excessive JavaScript execution +- Layout thrashing +- Unnecessary repaints +- Slow network requests +- Large bundle sizes + +--- + +## Network Analysis + +### Monitor Network Requests + +**Using chrome-devtools-mcp:** +```javascript +// Navigate to page +await useTool('navigate_page', { url: 'https://example.com' }); + +// Wait for all requests +await useTool('wait_for', { waitUntil: 'networkidle' }); + +// List all requests +const requests = await useTool('list_network_requests', { + resourceTypes: ['Document', 'Script', 'Stylesheet', 'Image', 'XHR', 'Fetch'], + pageSize: 100 +}); + +// Analyze specific request +for (const req of requests.requests) { + const details = await useTool('get_network_request', { + requestId: req.id + }); + + console.log({ + url: details.url, + method: details.method, + status: details.status, + size: details.encodedDataLength, + time: details.timing.receiveHeadersEnd - details.timing.requestTime, + cached: details.fromCache + }); +} +``` + +**Using Puppeteer:** +```javascript +const requests = []; + +// Capture all requests +page.on('request', (request) => { + requests.push({ + url: request.url(), + method: request.method(), + resourceType: request.resourceType(), + headers: request.headers() + }); +}); + +// Capture responses +page.on('response', (response) => { + const request = response.request(); + console.log({ + url: response.url(), + status: response.status(), + size: response.headers()['content-length'], + cached: response.fromCache(), + timing: response.timing() + }); +}); + +await page.goto('https://example.com'); +``` + +### Network Performance Metrics + +**Calculate Total Page Weight:** +```javascript +let totalBytes = 0; +let resourceCounts = {}; + +page.on('response', async (response) => { + const type = response.request().resourceType(); + const buffer = await response.buffer(); + + totalBytes += buffer.length; + resourceCounts[type] = (resourceCounts[type] || 0) + 1; +}); + +await page.goto('https://example.com'); + +console.log('Total size:', (totalBytes / 1024 / 1024).toFixed(2), 'MB'); +console.log('Resources:', resourceCounts); +``` + +**Identify Slow Requests:** +```javascript +page.on('response', (response) => { + const timing = response.timing(); + const totalTime = timing.receiveHeadersEnd - timing.requestTime; + + if (totalTime > 1000) { // Slower than 1 second + console.log('Slow request:', { + url: response.url(), + time: totalTime.toFixed(2) + 'ms', + size: response.headers()['content-length'] + }); + } +}); +``` + +### Network Throttling + +**Simulate Slow Connection:** +```javascript +// Using chrome-devtools-mcp +await useTool('emulate_network', { + throttlingOption: 'Slow 3G' // or 'Fast 3G', 'Slow 4G' +}); + +// Using Puppeteer +const client = await page.createCDPSession(); +await client.send('Network.emulateNetworkConditions', { + offline: false, + downloadThroughput: 400 * 1024 / 8, // 400 Kbps + uploadThroughput: 400 * 1024 / 8, + latency: 2000 // 2000ms RTT +}); +``` + +--- + +## JavaScript Performance + +### Identify Long Tasks + +**Using Performance Observer:** +```javascript +await page.evaluate(() => { + return new Promise((resolve) => { + const longTasks = []; + + const observer = new PerformanceObserver((list) => { + list.getEntries().forEach((entry) => { + longTasks.push({ + name: entry.name, + duration: entry.duration, + startTime: entry.startTime + }); + }); + }); + + observer.observe({ entryTypes: ['longtask'] }); + + // Collect for 10 seconds + setTimeout(() => { + observer.disconnect(); + resolve(longTasks); + }, 10000); + }); +}); +``` + +### CPU Profiling + +**Using Puppeteer:** +```javascript +// Start CPU profiling +const client = await page.createCDPSession(); +await client.send('Profiler.enable'); +await client.send('Profiler.start'); + +// Navigate and interact +await page.goto('https://example.com'); +await page.click('.button'); + +// Stop profiling +const { profile } = await client.send('Profiler.stop'); + +// Analyze profile (flame graph data) +// Import into Chrome DevTools for visualization +``` + +### JavaScript Coverage + +**Identify Unused Code:** +```javascript +// Start coverage +await Promise.all([ + page.coverage.startJSCoverage(), + page.coverage.startCSSCoverage() +]); + +// Navigate +await page.goto('https://example.com'); + +// Stop coverage +const [jsCoverage, cssCoverage] = await Promise.all([ + page.coverage.stopJSCoverage(), + page.coverage.stopCSSCoverage() +]); + +// Calculate unused bytes +function calculateUnusedBytes(coverage) { + let usedBytes = 0; + let totalBytes = 0; + + for (const entry of coverage) { + totalBytes += entry.text.length; + for (const range of entry.ranges) { + usedBytes += range.end - range.start - 1; + } + } + + return { + usedBytes, + totalBytes, + unusedBytes: totalBytes - usedBytes, + unusedPercentage: ((totalBytes - usedBytes) / totalBytes * 100).toFixed(2) + }; +} + +console.log('JS Coverage:', calculateUnusedBytes(jsCoverage)); +console.log('CSS Coverage:', calculateUnusedBytes(cssCoverage)); +``` + +### Bundle Size Analysis + +**Analyze JavaScript Bundles:** +```javascript +page.on('response', async (response) => { + const url = response.url(); + const type = response.request().resourceType(); + + if (type === 'script') { + const buffer = await response.buffer(); + const size = buffer.length; + + console.log({ + url: url.split('/').pop(), + size: (size / 1024).toFixed(2) + ' KB', + gzipped: response.headers()['content-encoding'] === 'gzip' + }); + } +}); +``` + +--- + +## Rendering Performance + +### Layout Thrashing Detection + +**Monitor Layout Recalculations:** +```javascript +// Using Performance Observer +await page.evaluate(() => { + return new Promise((resolve) => { + const measurements = []; + + const observer = new PerformanceObserver((list) => { + list.getEntries().forEach((entry) => { + if (entry.entryType === 'measure' && + entry.name.includes('layout')) { + measurements.push({ + name: entry.name, + duration: entry.duration, + startTime: entry.startTime + }); + } + }); + }); + + observer.observe({ entryTypes: ['measure'] }); + + setTimeout(() => { + observer.disconnect(); + resolve(measurements); + }, 5000); + }); +}); +``` + +### Paint and Composite Metrics + +**Get Paint Metrics:** +```javascript +const paintMetrics = await page.evaluate(() => { + const paints = performance.getEntriesByType('paint'); + return { + firstPaint: paints.find(p => p.name === 'first-paint')?.startTime, + firstContentfulPaint: paints.find(p => p.name === 'first-contentful-paint')?.startTime + }; +}); +``` + +### Frame Rate Analysis + +**Monitor FPS:** +```javascript +await page.evaluate(() => { + return new Promise((resolve) => { + let frames = 0; + let lastTime = performance.now(); + + function countFrames() { + frames++; + requestAnimationFrame(countFrames); + } + + countFrames(); + + setTimeout(() => { + const now = performance.now(); + const elapsed = (now - lastTime) / 1000; + const fps = frames / elapsed; + resolve(fps); + }, 5000); + }); +}); +``` + +### Layout Shifts (CLS) + +**Track Individual Shifts:** +```javascript +await page.evaluate(() => { + return new Promise((resolve) => { + const shifts = []; + let totalCLS = 0; + + const observer = new PerformanceObserver((list) => { + list.getEntries().forEach((entry) => { + if (!entry.hadRecentInput) { + totalCLS += entry.value; + shifts.push({ + value: entry.value, + time: entry.startTime, + elements: entry.sources?.map(s => s.node) + }); + } + }); + }); + + observer.observe({ entryTypes: ['layout-shift'] }); + + setTimeout(() => { + observer.disconnect(); + resolve({ totalCLS, shifts }); + }, 10000); + }); +}); +``` + +--- + +## Memory Analysis + +### Memory Metrics + +**Get Memory Usage:** +```javascript +// Using chrome-devtools-mcp +await useTool('evaluate_script', { + expression: ` + ({ + usedJSHeapSize: performance.memory?.usedJSHeapSize, + totalJSHeapSize: performance.memory?.totalJSHeapSize, + jsHeapSizeLimit: performance.memory?.jsHeapSizeLimit + }) + `, + returnByValue: true +}); + +// Using Puppeteer +const metrics = await page.metrics(); +console.log({ + jsHeapUsed: (metrics.JSHeapUsedSize / 1024 / 1024).toFixed(2) + ' MB', + jsHeapTotal: (metrics.JSHeapTotalSize / 1024 / 1024).toFixed(2) + ' MB', + domNodes: metrics.Nodes, + documents: metrics.Documents, + jsEventListeners: metrics.JSEventListeners +}); +``` + +### Memory Leak Detection + +**Monitor Memory Over Time:** +```javascript +async function detectMemoryLeak(page, duration = 30000) { + const samples = []; + const interval = 1000; // Sample every second + const samples_count = duration / interval; + + for (let i = 0; i < samples_count; i++) { + const metrics = await page.metrics(); + samples.push({ + time: i, + heapUsed: metrics.JSHeapUsedSize + }); + + await page.waitForTimeout(interval); + } + + // Analyze trend + const firstSample = samples[0].heapUsed; + const lastSample = samples[samples.length - 1].heapUsed; + const increase = ((lastSample - firstSample) / firstSample * 100).toFixed(2); + + return { + samples, + memoryIncrease: increase + '%', + possibleLeak: increase > 50 // > 50% increase indicates possible leak + }; +} + +const leakAnalysis = await detectMemoryLeak(page, 30000); +console.log('Memory Analysis:', leakAnalysis); +``` + +### Heap Snapshot + +**Capture Heap Snapshot:** +```javascript +const client = await page.createCDPSession(); + +// Take snapshot +await client.send('HeapProfiler.enable'); +const { result } = await client.send('HeapProfiler.takeHeapSnapshot'); + +// Snapshot is streamed in chunks +// Save to file or analyze programmatically +``` + +--- + +## Optimization Strategies + +### Image Optimization + +**Detect Unoptimized Images:** +```javascript +const images = await page.evaluate(() => { + const images = Array.from(document.querySelectorAll('img')); + return images.map(img => ({ + src: img.src, + naturalWidth: img.naturalWidth, + naturalHeight: img.naturalHeight, + displayWidth: img.width, + displayHeight: img.height, + oversized: img.naturalWidth > img.width * 1.5 || + img.naturalHeight > img.height * 1.5 + })); +}); + +const oversizedImages = images.filter(img => img.oversized); +console.log('Oversized images:', oversizedImages); +``` + +### Font Loading + +**Detect Render-Blocking Fonts:** +```javascript +const fonts = await page.evaluate(() => { + return Array.from(document.fonts).map(font => ({ + family: font.family, + weight: font.weight, + style: font.style, + status: font.status, + loaded: font.status === 'loaded' + })); +}); + +console.log('Fonts:', fonts); +``` + +### Third-Party Scripts + +**Measure Third-Party Impact:** +```javascript +const thirdPartyDomains = ['googletagmanager.com', 'facebook.net', 'doubleclick.net']; + +page.on('response', async (response) => { + const url = response.url(); + const isThirdParty = thirdPartyDomains.some(domain => url.includes(domain)); + + if (isThirdParty) { + const buffer = await response.buffer(); + console.log({ + url: url, + size: (buffer.length / 1024).toFixed(2) + ' KB', + type: response.request().resourceType() + }); + } +}); +``` + +### Critical Rendering Path + +**Identify Render-Blocking Resources:** +```javascript +await page.goto('https://example.com'); + +const renderBlockingResources = await page.evaluate(() => { + const resources = performance.getEntriesByType('resource'); + return resources.filter(resource => { + return (resource.initiatorType === 'link' && + resource.name.includes('.css')) || + (resource.initiatorType === 'script' && + !resource.name.includes('async')); + }).map(r => ({ + url: r.name, + duration: r.duration, + startTime: r.startTime + })); +}); + +console.log('Render-blocking resources:', renderBlockingResources); +``` + +### Lighthouse Integration + +**Run Lighthouse Audit:** +```javascript +import lighthouse from 'lighthouse'; +import { launch } from 'chrome-launcher'; + +// Launch Chrome +const chrome = await launch({ chromeFlags: ['--headless'] }); + +// Run Lighthouse +const { lhr } = await lighthouse('https://example.com', { + port: chrome.port, + onlyCategories: ['performance'] +}); + +// Get scores +console.log({ + performanceScore: lhr.categories.performance.score * 100, + metrics: { + FCP: lhr.audits['first-contentful-paint'].displayValue, + LCP: lhr.audits['largest-contentful-paint'].displayValue, + TBT: lhr.audits['total-blocking-time'].displayValue, + CLS: lhr.audits['cumulative-layout-shift'].displayValue, + SI: lhr.audits['speed-index'].displayValue + }, + opportunities: lhr.audits['opportunities'] +}); + +await chrome.kill(); +``` + +--- + +## Performance Budgets + +### Set Performance Budgets + +```javascript +const budgets = { + // Core Web Vitals + LCP: 2500, // ms + FID: 100, // ms + CLS: 0.1, // score + + // Other metrics + FCP: 1800, // ms + TTI: 3800, // ms + TBT: 300, // ms + + // Resource budgets + totalPageSize: 2 * 1024 * 1024, // 2 MB + jsSize: 500 * 1024, // 500 KB + cssSize: 100 * 1024, // 100 KB + imageSize: 1 * 1024 * 1024, // 1 MB + + // Request counts + totalRequests: 50, + jsRequests: 10, + cssRequests: 5 +}; + +async function checkBudgets(page, budgets) { + // Measure actual values + const vitals = await measureCoreWebVitals(page); + const resources = await analyzeResources(page); + + // Compare against budgets + const violations = []; + + if (vitals.LCP > budgets.LCP) { + violations.push(`LCP: ${vitals.LCP}ms exceeds budget of ${budgets.LCP}ms`); + } + + if (resources.totalSize > budgets.totalPageSize) { + violations.push(`Page size: ${resources.totalSize} exceeds budget of ${budgets.totalPageSize}`); + } + + // ... check other budgets + + return { + passed: violations.length === 0, + violations + }; +} +``` + +--- + +## Automated Performance Testing + +### CI/CD Integration + +```javascript +// performance-test.js +import puppeteer from 'puppeteer'; + +async function performanceTest(url) { + const browser = await puppeteer.launch(); + const page = await browser.newPage(); + + // Measure metrics + await page.goto(url, { waitUntil: 'networkidle2' }); + const metrics = await page.metrics(); + const vitals = await measureCoreWebVitals(page); + + await browser.close(); + + // Check against thresholds + const thresholds = { + LCP: 2500, + FID: 100, + CLS: 0.1, + jsHeapSize: 50 * 1024 * 1024 // 50 MB + }; + + const failed = []; + if (vitals.LCP > thresholds.LCP) failed.push('LCP'); + if (vitals.FID > thresholds.FID) failed.push('FID'); + if (vitals.CLS > thresholds.CLS) failed.push('CLS'); + if (metrics.JSHeapUsedSize > thresholds.jsHeapSize) failed.push('Memory'); + + if (failed.length > 0) { + console.error('Performance test failed:', failed); + process.exit(1); + } + + console.log('Performance test passed'); +} + +performanceTest(process.env.TEST_URL); +``` + +--- + +## Best Practices + +### Performance Testing Checklist + +1. **Measure Multiple Times** + - Run tests 3-5 times + - Use median values + - Account for variance + +2. **Test Different Conditions** + - Fast 3G + - Slow 3G + - Offline + - CPU throttling + +3. **Test Different Devices** + - Mobile (low-end) + - Mobile (high-end) + - Desktop + - Tablet + +4. **Monitor Over Time** + - Track metrics in CI/CD + - Set up alerts for regressions + - Create performance dashboards + +5. **Focus on User Experience** + - Prioritize Core Web Vitals + - Test real user journeys + - Consider perceived performance + +6. **Optimize Critical Path** + - Minimize render-blocking resources + - Defer non-critical JavaScript + - Optimize font loading + - Lazy load images + +--- + +## Resources + +- [Web.dev Performance](https://web.dev/performance/) +- [Chrome DevTools Performance](https://developer.chrome.com/docs/devtools/performance/) +- [Core Web Vitals](https://web.dev/vitals/) +- [Lighthouse](https://developer.chrome.com/docs/lighthouse/) +- [WebPageTest](https://www.webpagetest.org/) diff --git a/skills/chrome-devtools/references/puppeteer-reference.md b/skills/chrome-devtools/references/puppeteer-reference.md new file mode 100644 index 0000000..cbf7f6e --- /dev/null +++ b/skills/chrome-devtools/references/puppeteer-reference.md @@ -0,0 +1,953 @@ +# Puppeteer Quick Reference + +Complete guide to browser automation with Puppeteer - a high-level API over Chrome DevTools Protocol. + +## Table of Contents + +- [Setup](#setup) +- [Browser & Page Management](#browser--page-management) +- [Navigation](#navigation) +- [Element Interaction](#element-interaction) +- [JavaScript Execution](#javascript-execution) +- [Screenshots & PDFs](#screenshots--pdfs) +- [Network Interception](#network-interception) +- [Device Emulation](#device-emulation) +- [Performance](#performance) +- [Common Patterns](#common-patterns) + +--- + +## Setup + +### Installation + +```bash +# Install Puppeteer +npm install puppeteer + +# Install core only (bring your own Chrome) +npm install puppeteer-core +``` + +### Basic Usage + +```javascript +import puppeteer from 'puppeteer'; + +// Launch browser +const browser = await puppeteer.launch({ + headless: true, + args: ['--no-sandbox'] +}); + +// Open page +const page = await browser.newPage(); + +// Navigate +await page.goto('https://example.com'); + +// Do work... + +// Cleanup +await browser.close(); +``` + +--- + +## Browser & Page Management + +### Launch Browser + +```javascript +const browser = await puppeteer.launch({ + // Visibility + headless: false, // Show browser UI + headless: 'new', // New headless mode (Chrome 112+) + + // Chrome location + executablePath: '/path/to/chrome', + channel: 'chrome', // or 'chrome-canary', 'chrome-beta' + + // Browser context + userDataDir: './user-data', // Persistent profile + + // Window size + defaultViewport: { + width: 1920, + height: 1080, + deviceScaleFactor: 1, + isMobile: false + }, + + // Advanced options + args: [ + '--no-sandbox', + '--disable-setuid-sandbox', + '--disable-dev-shm-usage', + '--disable-web-security', + '--disable-features=IsolateOrigins', + '--disable-site-isolation-trials', + '--start-maximized' + ], + + // Debugging + devtools: true, // Open DevTools automatically + slowMo: 250, // Slow down by 250ms per action + + // Network + proxy: { + server: 'http://proxy.com:8080' + } +}); +``` + +### Connect to Running Browser + +```javascript +// Launch Chrome with debugging +// google-chrome --remote-debugging-port=9222 + +const browser = await puppeteer.connect({ + browserURL: 'http://localhost:9222', + // or browserWSEndpoint: 'ws://localhost:9222/devtools/browser/...' +}); +``` + +### Page Management + +```javascript +// Create new page +const page = await browser.newPage(); + +// Get all pages +const pages = await browser.pages(); + +// Close page +await page.close(); + +// Multiple pages +const page1 = await browser.newPage(); +const page2 = await browser.newPage(); + +// Switch between pages +await page1.bringToFront(); +``` + +### Browser Context (Incognito) + +```javascript +// Create isolated context +const context = await browser.createBrowserContext(); +const page = await context.newPage(); + +// Cleanup context +await context.close(); +``` + +--- + +## Navigation + +### Basic Navigation + +```javascript +// Navigate to URL +await page.goto('https://example.com'); + +// Navigate with options +await page.goto('https://example.com', { + waitUntil: 'networkidle2', // or 'load', 'domcontentloaded', 'networkidle0' + timeout: 30000 // Max wait time (ms) +}); + +// Reload page +await page.reload({ waitUntil: 'networkidle2' }); + +// Navigation history +await page.goBack(); +await page.goForward(); + +// Wait for navigation +await page.waitForNavigation({ + waitUntil: 'networkidle2' +}); +``` + +### Wait Until Options + +- `load` - Wait for load event +- `domcontentloaded` - Wait for DOMContentLoaded event +- `networkidle0` - Wait until no network connections for 500ms +- `networkidle2` - Wait until max 2 network connections for 500ms + +--- + +## Element Interaction + +### Selectors + +```javascript +// CSS selectors +await page.$('#id'); +await page.$('.class'); +await page.$('div > p'); + +// XPath +await page.$x('//button[text()="Submit"]'); + +// Get all matching elements +await page.$$('.item'); +await page.$$x('//div[@class="item"]'); +``` + +### Click Elements + +```javascript +// Click by selector +await page.click('.button'); + +// Click with options +await page.click('.button', { + button: 'left', // or 'right', 'middle' + clickCount: 1, // 2 for double-click + delay: 100 // Delay between mousedown and mouseup +}); + +// ElementHandle click +const button = await page.$('.button'); +await button.click(); +``` + +### Type Text + +```javascript +// Type into input +await page.type('#search', 'query text'); + +// Type with delay +await page.type('#search', 'slow typing', { delay: 100 }); + +// Clear and type +await page.$eval('#search', el => el.value = ''); +await page.type('#search', 'new text'); +``` + +### Form Interaction + +```javascript +// Fill input +await page.type('#username', 'john@example.com'); +await page.type('#password', 'secret123'); + +// Select dropdown option +await page.select('#country', 'US'); // By value +await page.select('#country', 'USA', 'UK'); // Multiple + +// Check/uncheck checkbox +await page.click('input[type="checkbox"]'); + +// Choose radio button +await page.click('input[value="option2"]'); + +// Upload file +const input = await page.$('input[type="file"]'); +await input.uploadFile('/path/to/file.pdf'); + +// Submit form +await page.click('button[type="submit"]'); +await page.waitForNavigation(); +``` + +### Hover & Focus + +```javascript +// Hover over element +await page.hover('.menu-item'); + +// Focus element +await page.focus('#input'); + +// Blur +await page.$eval('#input', el => el.blur()); +``` + +### Drag & Drop + +```javascript +const source = await page.$('.draggable'); +const target = await page.$('.drop-zone'); + +await source.drag(target); +await source.drop(target); +``` + +--- + +## JavaScript Execution + +### Evaluate in Page Context + +```javascript +// Execute JavaScript +const title = await page.evaluate(() => document.title); + +// With arguments +const text = await page.evaluate( + (selector) => document.querySelector(selector).textContent, + '.heading' +); + +// Return complex data +const data = await page.evaluate(() => ({ + title: document.title, + url: location.href, + cookies: document.cookie +})); + +// With ElementHandle +const element = await page.$('.button'); +const text = await page.evaluate(el => el.textContent, element); +``` + +### Query & Modify DOM + +```javascript +// Get element property +const value = await page.$eval('#input', el => el.value); + +// Get multiple elements +const items = await page.$$eval('.item', elements => + elements.map(el => el.textContent) +); + +// Modify element +await page.$eval('#input', (el, value) => { + el.value = value; +}, 'new value'); + +// Add class +await page.$eval('.element', el => el.classList.add('active')); +``` + +### Expose Functions + +```javascript +// Expose Node.js function to page +await page.exposeFunction('md5', (text) => + crypto.createHash('md5').update(text).digest('hex') +); + +// Call from page context +const hash = await page.evaluate(async () => { + return await window.md5('hello world'); +}); +``` + +--- + +## Screenshots & PDFs + +### Screenshots + +```javascript +// Full page screenshot +await page.screenshot({ + path: 'screenshot.png', + fullPage: true +}); + +// Viewport screenshot +await page.screenshot({ + path: 'viewport.png', + fullPage: false +}); + +// Element screenshot +const element = await page.$('.chart'); +await element.screenshot({ + path: 'chart.png' +}); + +// Screenshot options +await page.screenshot({ + path: 'page.png', + type: 'png', // or 'jpeg', 'webp' + quality: 80, // JPEG quality (0-100) + clip: { // Crop region + x: 0, + y: 0, + width: 500, + height: 500 + }, + omitBackground: true // Transparent background +}); + +// Screenshot to buffer +const buffer = await page.screenshot(); +``` + +### PDF Generation + +```javascript +// Generate PDF +await page.pdf({ + path: 'page.pdf', + format: 'A4', // or 'Letter', 'Legal', etc. + printBackground: true, + margin: { + top: '1cm', + right: '1cm', + bottom: '1cm', + left: '1cm' + } +}); + +// Custom page size +await page.pdf({ + path: 'custom.pdf', + width: '8.5in', + height: '11in', + landscape: true +}); + +// Header and footer +await page.pdf({ + path: 'report.pdf', + displayHeaderFooter: true, + headerTemplate: '
Header
', + footerTemplate: '
Page
' +}); +``` + +--- + +## Network Interception + +### Request Interception + +```javascript +// Enable request interception +await page.setRequestInterception(true); + +// Intercept requests +page.on('request', (request) => { + // Block specific resource types + if (request.resourceType() === 'image') { + request.abort(); + } + // Block URLs + else if (request.url().includes('ads')) { + request.abort(); + } + // Modify request + else if (request.url().includes('api')) { + request.continue({ + headers: { + ...request.headers(), + 'Authorization': 'Bearer token' + } + }); + } + // Continue normally + else { + request.continue(); + } +}); +``` + +### Mock Responses + +```javascript +await page.setRequestInterception(true); + +page.on('request', (request) => { + if (request.url().includes('/api/user')) { + request.respond({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + id: 1, + name: 'Mock User' + }) + }); + } else { + request.continue(); + } +}); +``` + +### Monitor Network + +```javascript +// Track requests +page.on('request', (request) => { + console.log('Request:', request.method(), request.url()); +}); + +// Track responses +page.on('response', (response) => { + console.log('Response:', response.status(), response.url()); +}); + +// Track failed requests +page.on('requestfailed', (request) => { + console.log('Failed:', request.failure().errorText, request.url()); +}); + +// Get response body +page.on('response', async (response) => { + if (response.url().includes('/api/data')) { + const json = await response.json(); + console.log('API Data:', json); + } +}); +``` + +--- + +## Device Emulation + +### Predefined Devices + +```javascript +import { devices } from 'puppeteer'; + +// Emulate iPhone +const iPhone = devices['iPhone 13 Pro']; +await page.emulate(iPhone); + +// Common devices +const iPad = devices['iPad Pro']; +const pixel = devices['Pixel 5']; +const galaxy = devices['Galaxy S9+']; + +// Navigate after emulation +await page.goto('https://example.com'); +``` + +### Custom Device + +```javascript +await page.emulate({ + viewport: { + width: 375, + height: 812, + deviceScaleFactor: 3, + isMobile: true, + hasTouch: true, + isLandscape: false + }, + userAgent: 'Mozilla/5.0 (iPhone; CPU iPhone OS 14_0 like Mac OS X)...' +}); +``` + +### Viewport Only + +```javascript +await page.setViewport({ + width: 1920, + height: 1080, + deviceScaleFactor: 1 +}); +``` + +### Geolocation + +```javascript +// Set geolocation +await page.setGeolocation({ + latitude: 37.7749, + longitude: -122.4194, + accuracy: 100 +}); + +// Grant permissions +const context = browser.defaultBrowserContext(); +await context.overridePermissions('https://example.com', ['geolocation']); +``` + +### Timezone & Locale + +```javascript +// Set timezone +await page.emulateTimezone('America/New_York'); + +// Set locale +await page.emulateMediaType('screen'); +await page.evaluateOnNewDocument(() => { + Object.defineProperty(navigator, 'language', { + get: () => 'en-US' + }); +}); +``` + +--- + +## Performance + +### CPU & Network Throttling + +```javascript +// CPU throttling +const client = await page.createCDPSession(); +await client.send('Emulation.setCPUThrottlingRate', { rate: 4 }); + +// Network throttling +await page.emulateNetworkConditions({ + offline: false, + downloadThroughput: 1.5 * 1024 * 1024 / 8, // 1.5 Mbps + uploadThroughput: 750 * 1024 / 8, // 750 Kbps + latency: 40 // 40ms RTT +}); + +// Predefined profiles +await page.emulateNetworkConditions( + puppeteer.networkConditions['Fast 3G'] +); + +// Disable throttling +await page.emulateNetworkConditions({ + offline: false, + downloadThroughput: -1, + uploadThroughput: -1, + latency: 0 +}); +``` + +### Performance Metrics + +```javascript +// Get metrics +const metrics = await page.metrics(); +console.log(metrics); +// { +// Timestamp, Documents, Frames, JSEventListeners, +// Nodes, LayoutCount, RecalcStyleCount, +// LayoutDuration, RecalcStyleDuration, +// ScriptDuration, TaskDuration, +// JSHeapUsedSize, JSHeapTotalSize +// } +``` + +### Performance Tracing + +```javascript +// Start tracing +await page.tracing.start({ + path: 'trace.json', + categories: [ + 'devtools.timeline', + 'disabled-by-default-devtools.timeline' + ] +}); + +// Navigate +await page.goto('https://example.com'); + +// Stop tracing +await page.tracing.stop(); + +// Analyze trace in chrome://tracing +``` + +### Coverage (Code Usage) + +```javascript +// Start JS coverage +await page.coverage.startJSCoverage(); + +// Start CSS coverage +await page.coverage.startCSSCoverage(); + +// Navigate +await page.goto('https://example.com'); + +// Stop and get coverage +const jsCoverage = await page.coverage.stopJSCoverage(); +const cssCoverage = await page.coverage.stopCSSCoverage(); + +// Calculate unused bytes +let totalBytes = 0; +let usedBytes = 0; +for (const entry of [...jsCoverage, ...cssCoverage]) { + totalBytes += entry.text.length; + for (const range of entry.ranges) { + usedBytes += range.end - range.start - 1; + } +} + +console.log(`Used: ${usedBytes / totalBytes * 100}%`); +``` + +--- + +## Common Patterns + +### Wait for Elements + +```javascript +// Wait for selector +await page.waitForSelector('.element', { + visible: true, + timeout: 5000 +}); + +// Wait for XPath +await page.waitForXPath('//button[text()="Submit"]'); + +// Wait for function +await page.waitForFunction( + () => document.querySelector('.loading') === null, + { timeout: 10000 } +); + +// Wait for timeout +await page.waitForTimeout(2000); +``` + +### Handle Dialogs + +```javascript +// Alert, confirm, prompt +page.on('dialog', async (dialog) => { + console.log(dialog.type(), dialog.message()); + + // Accept + await dialog.accept(); + // or reject + // await dialog.dismiss(); + // or provide input for prompt + // await dialog.accept('input text'); +}); +``` + +### Handle Downloads + +```javascript +// Set download path +const client = await page.createCDPSession(); +await client.send('Page.setDownloadBehavior', { + behavior: 'allow', + downloadPath: '/path/to/downloads' +}); + +// Trigger download +await page.click('a[download]'); +``` + +### Multiple Pages (Tabs) + +```javascript +// Listen for new pages +browser.on('targetcreated', async (target) => { + if (target.type() === 'page') { + const newPage = await target.page(); + console.log('New page opened:', newPage.url()); + } +}); + +// Click link that opens new tab +const [newPage] = await Promise.all([ + new Promise(resolve => browser.once('targetcreated', target => resolve(target.page()))), + page.click('a[target="_blank"]') +]); + +console.log('New page URL:', newPage.url()); +``` + +### Frames (iframes) + +```javascript +// Get all frames +const frames = page.frames(); + +// Find frame by name +const frame = page.frames().find(f => f.name() === 'myframe'); + +// Find frame by URL +const frame = page.frames().find(f => f.url().includes('example.com')); + +// Main frame +const mainFrame = page.mainFrame(); + +// Interact with frame +await frame.click('.button'); +await frame.type('#input', 'text'); +``` + +### Infinite Scroll + +```javascript +async function autoScroll(page) { + await page.evaluate(async () => { + await new Promise((resolve) => { + let totalHeight = 0; + const distance = 100; + const timer = setInterval(() => { + const scrollHeight = document.body.scrollHeight; + window.scrollBy(0, distance); + totalHeight += distance; + + if (totalHeight >= scrollHeight) { + clearInterval(timer); + resolve(); + } + }, 100); + }); + }); +} + +await autoScroll(page); +``` + +### Cookies + +```javascript +// Get cookies +const cookies = await page.cookies(); + +// Set cookies +await page.setCookie({ + name: 'session', + value: 'abc123', + domain: 'example.com', + path: '/', + httpOnly: true, + secure: true, + sameSite: 'Strict' +}); + +// Delete cookies +await page.deleteCookie({ name: 'session' }); +``` + +### Local Storage + +```javascript +// Set localStorage +await page.evaluate(() => { + localStorage.setItem('key', 'value'); +}); + +// Get localStorage +const value = await page.evaluate(() => { + return localStorage.getItem('key'); +}); + +// Clear localStorage +await page.evaluate(() => localStorage.clear()); +``` + +### Error Handling + +```javascript +try { + await page.goto('https://example.com', { + waitUntil: 'networkidle2', + timeout: 30000 + }); +} catch (error) { + if (error.name === 'TimeoutError') { + console.error('Page load timeout'); + } else { + console.error('Navigation failed:', error); + } + + // Take screenshot on error + await page.screenshot({ path: 'error.png' }); +} +``` + +### Stealth Mode (Avoid Detection) + +```javascript +// Hide automation indicators +await page.evaluateOnNewDocument(() => { + // Override navigator.webdriver + Object.defineProperty(navigator, 'webdriver', { + get: () => false + }); + + // Mock chrome object + window.chrome = { + runtime: {} + }; + + // Mock permissions + const originalQuery = window.navigator.permissions.query; + window.navigator.permissions.query = (parameters) => ( + parameters.name === 'notifications' ? + Promise.resolve({ state: 'granted' }) : + originalQuery(parameters) + ); +}); + +// Set realistic user agent +await page.setUserAgent( + 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' +); +``` + +--- + +## Debugging Tips + +### Take Screenshots on Error + +```javascript +page.on('pageerror', async (error) => { + console.error('Page error:', error); + await page.screenshot({ path: `error-${Date.now()}.png` }); +}); +``` + +### Console Logging + +```javascript +// Forward console to Node +page.on('console', (msg) => { + console.log('PAGE LOG:', msg.text()); +}); +``` + +### Slow Down Execution + +```javascript +const browser = await puppeteer.launch({ + slowMo: 250 // 250ms delay between actions +}); +``` + +### Keep Browser Open + +```javascript +const browser = await puppeteer.launch({ + headless: false, + devtools: true +}); + +// Prevent auto-close +await page.evaluate(() => debugger); +``` + +--- + +## Best Practices + +1. **Always close browser:** Use try/finally or process cleanup +2. **Wait appropriately:** Use waitForSelector, not setTimeout +3. **Handle errors:** Wrap navigation in try/catch +4. **Optimize selectors:** Use specific selectors for reliability +5. **Avoid race conditions:** Wait for navigation after clicks +6. **Reuse pages:** Don't create new pages unnecessarily +7. **Set timeouts:** Always specify reasonable timeouts +8. **Clean up:** Close unused pages and contexts + +--- + +## Resources + +- [Puppeteer Documentation](https://pptr.dev/) +- [Puppeteer API](https://pptr.dev/api) +- [Puppeteer Examples](https://github.com/puppeteer/puppeteer/tree/main/examples) +- [Awesome Puppeteer](https://github.com/transitive-bullshit/awesome-puppeteer) diff --git a/skills/chrome-devtools/scripts/README.md b/skills/chrome-devtools/scripts/README.md new file mode 100644 index 0000000..f44fa6e --- /dev/null +++ b/skills/chrome-devtools/scripts/README.md @@ -0,0 +1,213 @@ +# Chrome DevTools Scripts + +CLI scripts for browser automation using Puppeteer. + +**CRITICAL**: Always check `pwd` before running scripts. + +## Installation + +### Quick Install + +```bash +pwd # Should show current working directory +cd .claude/skills/chrome-devtools/scripts +./install.sh # Auto-checks dependencies and installs +``` + +### Manual Installation + +**Linux/WSL** - Install system dependencies first: +```bash +./install-deps.sh # Auto-detects OS (Ubuntu, Debian, Fedora, etc.) +``` + +Or manually: +```bash +sudo apt-get install -y libnss3 libnspr4 libasound2t64 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 libgbm1 +``` + +**All platforms** - Install Node dependencies: +```bash +npm install +``` + +## Scripts + +**CRITICAL**: Always check `pwd` before running scripts. + +### navigate.js +Navigate to a URL. + +```bash +node navigate.js --url https://example.com [--wait-until networkidle2] [--timeout 30000] +``` + +### screenshot.js +Take a screenshot with automatic compression. + +**Important**: Always save screenshots to `./docs/screenshots` directory. + +```bash +node screenshot.js --output screenshot.png [--url https://example.com] [--full-page true] [--selector .element] [--max-size 5] [--no-compress] +``` + +**Automatic Compression**: Screenshots >5MB are automatically compressed using ImageMagick to ensure compatibility with Gemini API and Claude Code. Install ImageMagick for this feature: +- macOS: `brew install imagemagick` +- Linux: `sudo apt-get install imagemagick` + +Options: +- `--max-size N` - Custom size threshold in MB (default: 5) +- `--no-compress` - Disable automatic compression +- `--format png|jpeg` - Output format (default: png) +- `--quality N` - JPEG quality 0-100 (default: auto) + +### click.js +Click an element. + +```bash +node click.js --selector ".button" [--url https://example.com] [--wait-for ".result"] +``` + +### fill.js +Fill form fields. + +```bash +node fill.js --selector "#input" --value "text" [--url https://example.com] [--clear true] +``` + +### evaluate.js +Execute JavaScript in page context. + +```bash +node evaluate.js --script "document.title" [--url https://example.com] +``` + +### snapshot.js +Get DOM snapshot with interactive elements. + +```bash +node snapshot.js [--url https://example.com] [--output snapshot.json] +``` + +### console.js +Monitor console messages. + +```bash +node console.js --url https://example.com [--types error,warn] [--duration 5000] +``` + +### network.js +Monitor network requests. + +```bash +node network.js --url https://example.com [--types xhr,fetch] [--output requests.json] +``` + +### performance.js +Measure performance metrics and record trace. + +```bash +node performance.js --url https://example.com [--trace trace.json] [--metrics] [--resources true] +``` + +## Common Options + +- `--headless false` - Show browser window +- `--close false` - Keep browser open +- `--timeout 30000` - Set timeout in milliseconds +- `--wait-until networkidle2` - Wait strategy (load, domcontentloaded, networkidle0, networkidle2) + +## Selector Support + +Scripts that accept `--selector` (click.js, fill.js, screenshot.js) support both **CSS** and **XPath** selectors. + +### CSS Selectors (Default) + +```bash +# Element tag +node click.js --selector "button" --url https://example.com + +# Class selector +node click.js --selector ".btn-submit" --url https://example.com + +# ID selector +node fill.js --selector "#email" --value "user@example.com" --url https://example.com + +# Attribute selector +node click.js --selector 'button[type="submit"]' --url https://example.com + +# Complex selector +node screenshot.js --selector "div.container > button.btn-primary" --output btn.png +``` + +### XPath Selectors + +XPath selectors start with `/` or `(//` and are automatically detected: + +```bash +# Text matching - exact +node click.js --selector '//button[text()="Submit"]' --url https://example.com + +# Text matching - contains +node click.js --selector '//button[contains(text(),"Submit")]' --url https://example.com + +# Attribute matching +node fill.js --selector '//input[@type="email"]' --value "user@example.com" + +# Multiple conditions +node click.js --selector '//button[@type="submit" and contains(text(),"Save")]' + +# Descendant selection +node screenshot.js --selector '//div[@class="modal"]//button[@class="close"]' --output modal.png + +# Nth element +node click.js --selector '(//button)[2]' # Second button on page +``` + +### Discovering Selectors + +Use `snapshot.js` to discover correct selectors: + +```bash +# Get all interactive elements +node snapshot.js --url https://example.com | jq '.elements[]' + +# Find buttons +node snapshot.js --url https://example.com | jq '.elements[] | select(.tagName=="BUTTON")' + +# Find inputs +node snapshot.js --url https://example.com | jq '.elements[] | select(.tagName=="INPUT")' +``` + +### Security + +XPath selectors are validated to prevent injection attacks. The following patterns are blocked: +- `javascript:` +- ` { + describe('CSS Selectors', () => { + it('should detect simple CSS selectors', () => { + const result = parseSelector('button'); + assert.strictEqual(result.type, 'css'); + assert.strictEqual(result.selector, 'button'); + }); + + it('should detect class selectors', () => { + const result = parseSelector('.btn-submit'); + assert.strictEqual(result.type, 'css'); + assert.strictEqual(result.selector, '.btn-submit'); + }); + + it('should detect ID selectors', () => { + const result = parseSelector('#email-input'); + assert.strictEqual(result.type, 'css'); + assert.strictEqual(result.selector, '#email-input'); + }); + + it('should detect attribute selectors', () => { + const result = parseSelector('button[type="submit"]'); + assert.strictEqual(result.type, 'css'); + assert.strictEqual(result.selector, 'button[type="submit"]'); + }); + + it('should detect complex CSS selectors', () => { + const result = parseSelector('div.container > button.btn-primary:hover'); + assert.strictEqual(result.type, 'css'); + }); + }); + + describe('XPath Selectors', () => { + it('should detect absolute XPath', () => { + const result = parseSelector('/html/body/button'); + assert.strictEqual(result.type, 'xpath'); + assert.strictEqual(result.selector, '/html/body/button'); + }); + + it('should detect relative XPath', () => { + const result = parseSelector('//button'); + assert.strictEqual(result.type, 'xpath'); + assert.strictEqual(result.selector, '//button'); + }); + + it('should detect XPath with text matching', () => { + const result = parseSelector('//button[text()="Click Me"]'); + assert.strictEqual(result.type, 'xpath'); + }); + + it('should detect XPath with contains', () => { + const result = parseSelector('//button[contains(text(),"Submit")]'); + assert.strictEqual(result.type, 'xpath'); + }); + + it('should detect XPath with attributes', () => { + const result = parseSelector('//input[@type="email"]'); + assert.strictEqual(result.type, 'xpath'); + }); + + it('should detect grouped XPath', () => { + const result = parseSelector('(//button)[1]'); + assert.strictEqual(result.type, 'xpath'); + }); + }); + + describe('Security Validation', () => { + it('should block javascript: injection', () => { + assert.throws( + () => parseSelector('//button[@onclick="javascript:alert(1)"]'), + /XPath injection detected.*javascript:/i + ); + }); + + it('should block ")]'), + /XPath injection detected.* + ``` + +3. **Manual**: [from agent 3] + Download and include in project + +## Core Concepts + +[Synthesized from agents 2 & 4] +The library is built around three main concepts: + +1. **Components**: [definition from agent 2] +2. **State**: [definition from agent 4] +3. **Effects**: [definition from agent 2] + +## Examples + +[From agents 3 & 5, deduplicated] +... +``` + +Benefits: +- Organized by topic +- Deduplicated +- Clear narrative +- Easy to scan + +### Synthesis Techniques + +**Deduplication:** +``` +Agent 1: "Install with npm install foo" +Agent 2: "You can install using npm: npm install foo" +→ Synthesized: "Install: `npm install foo`" +``` + +**Prioritization:** +``` +Agent 1: Basic usage example +Agent 2: Basic usage example (same) +Agent 3: Advanced usage example +→ Keep: Basic (from agent 1) + Advanced (from agent 3) +``` + +**Organization:** +``` +Agents returned mixed information: +- Installation steps +- Configuration +- Usage example +- Installation requirements +- More usage examples + +→ Reorganize: +1. Installation (requirements + steps) +2. Configuration +3. Usage (all examples together) +``` + +## 7. Time Management + +### Why + +- **User experience**: Fast results +- **Resource efficiency**: Don't waste compute +- **Fail fast**: Quickly try alternatives +- **Practical limits**: Avoid hanging + +### Timeouts + +**Set explicit timeouts:** +``` +WebSearch: 30 seconds +WebFetch: 60 seconds +Repository clone: 5 minutes +Repomix processing: 10 minutes +Explorer agent: 5 minutes per URL +Researcher agent: 10 minutes +``` + +### Time Budgets + +**Simple query (single library, latest version):** +``` +Target: <2 minutes total + +Phase 1 (Discovery): 30 seconds +- llms.txt search: 15 seconds +- Fetch llms.txt: 15 seconds + +Phase 2 (Exploration): 60 seconds +- Launch agents: 5 seconds +- Agents fetch URLs: 60 seconds (parallel) + +Phase 3 (Aggregation): 30 seconds +- Synthesize results +- Format output + +Total: ~2 minutes +``` + +**Complex query (multiple versions, comparison):** +``` +Target: <5 minutes total + +Phase 1 (Discovery): 60 seconds +- Search both versions +- Fetch both llms.txt files + +Phase 2 (Exploration): 180 seconds +- Launch 6 agents (2 sets of 3) +- Parallel exploration + +Phase 3 (Comparison): 60 seconds +- Analyze differences +- Format side-by-side + +Total: ~5 minutes +``` + +### When to Extend Timeouts + +Acceptable to go longer when: +- User explicitly requests comprehensive analysis +- Repository is large but necessary +- Multiple fallbacks attempted +- User is informed of delay + +### When to Give Up + +Move to next method after: +- 3 failed attempts on same approach +- Timeout exceeded by 2x +- No progress for 30 seconds +- Error indicates permanent failure (404, auth required) + +## 8. Cache Findings + +### Why + +- **Speed**: Instant results for repeated requests +- **Efficiency**: Reduce network requests +- **Consistency**: Same results within session +- **Reliability**: Less dependent on network + +### What to Cache + +**High value (always cache):** +``` +- Repomix output (large, expensive to generate) +- llms.txt content (static, frequently referenced) +- Repository README (relatively static) +- Package registry metadata (changes rarely) +``` + +**Medium value (cache within session):** +``` +- Documentation page content +- Search results +- Repository structure +- Version lists +``` + +**Low value (don't cache):** +``` +- Real-time data (latest releases) +- User-specific content +- Time-sensitive information +``` + +### Cache Duration + +``` +Within conversation: +- All fetched content (reuse freely) + +Within session: +- Repomix output (until conversation ends) +- llms.txt content (until new version requested) + +Across sessions: +- Don't cache (start fresh each time) +``` + +### Cache Invalidation + +Refresh cache when: +``` +- User requests specific different version +- User says "get latest" or "refresh" +- Explicit time reference ("docs from today") +- Previous cache is from different library +``` + +### Implementation + +``` +# First request for library X +1. Fetch llms.txt +2. Store content in session variable +3. Use for processing + +# Second request for library X (same session) +1. Check if llms.txt cached +2. Reuse cached content +3. Skip redundant fetch + +# Request for library Y +1. Don't reuse library X cache +2. Fetch fresh for library Y +``` + +### Cache Hit Messages + +```markdown +ℹ️ Using cached llms.txt from 5 minutes ago. +To fetch fresh, say "refresh" or "get latest". +``` + +## Quick Reference Checklist + +### Before Starting + +- [ ] Identify library name clearly +- [ ] Confirm version (default: latest) +- [ ] Check if cached data available +- [ ] Plan method (llms.txt → repo → research) + +### During Discovery + +- [ ] Start with llms.txt search +- [ ] Verify source is official +- [ ] Check version matches requirement +- [ ] Set timeout for each operation +- [ ] Fall back quickly if method fails + +### During Exploration + +- [ ] Use parallel agents for 3+ URLs +- [ ] Launch all agents in single message +- [ ] Distribute workload evenly +- [ ] Monitor for errors/timeouts +- [ ] Be ready to retry or fallback + +### Before Presenting + +- [ ] Synthesize by topic (not by agent) +- [ ] Deduplicate repeated information +- [ ] Verify version is correct +- [ ] Include source attribution +- [ ] Note any limitations +- [ ] Format clearly +- [ ] Check completeness + +### Quality Gates + +Ask before presenting: +- [ ] Is information accurate? +- [ ] Are sources official? +- [ ] Does version match request? +- [ ] Are all key topics covered? +- [ ] Are limitations noted? +- [ ] Is methodology documented? +- [ ] Is output well-organized? diff --git a/skills/docs-seeker/references/documentation-sources.md b/skills/docs-seeker/references/documentation-sources.md new file mode 100644 index 0000000..81e4a1e --- /dev/null +++ b/skills/docs-seeker/references/documentation-sources.md @@ -0,0 +1,461 @@ +# Common Documentation Sources + +Reference guide for locating documentation across popular platforms and ecosystems. + +## context7.com Locations (PRIORITY) + +**ALWAYS try context7.com first for all libraries** + +### JavaScript/TypeScript Frameworks + +- **Astro**: https://context7.com/withastro/astro/llms.txt +- **Next.js**: https://context7.com/vercel/next.js/llms.txt +- **Remix**: https://context7.com/remix-run/remix/llms.txt +- **SvelteKit**: https://context7.com/sveltejs/kit/llms.txt +- **Nuxt**: https://context7.com/nuxt/nuxt/llms.txt + +### Frontend Libraries & UI + +- **React**: https://context7.com/facebook/react/llms.txt +- **Vue**: https://context7.com/vuejs/core/llms.txt +- **Svelte**: https://context7.com/sveltejs/svelte/llms.txt +- **shadcn/ui**: https://context7.com/shadcn-ui/ui/llms.txt +- **Radix UI**: https://context7.com/radix-ui/primitives/llms.txt + +### Backend/Full-stack + +- **Hono**: https://context7.com/honojs/hono/llms.txt +- **Fastify**: https://context7.com/fastify/fastify/llms.txt +- **tRPC**: https://context7.com/trpc/trpc/llms.txt + +### Build Tools + +- **Vite**: https://context7.com/vitejs/vite/llms.txt +- **Turbo**: https://context7.com/vercel/turbo/llms.txt + +### Databases/ORMs + +- **Prisma**: https://context7.com/prisma/prisma/llms.txt +- **Drizzle**: https://context7.com/drizzle-team/drizzle-orm/llms.txt + +### Authentication + +- **Better Auth**: https://context7.com/better-auth/better-auth/llms.txt +- **Auth.js**: https://context7.com/nextauthjs/next-auth/llms.txt + +### Image Processing + +- **ImageMagick**: https://context7.com/imagick/imagick/llms.txt + +### Topic-Specific Examples + +- **shadcn/ui date components**: https://context7.com/shadcn-ui/ui/llms.txt?topic=date +- **shadcn/ui buttons**: https://context7.com/shadcn-ui/ui/llms.txt?topic=button +- **Next.js caching**: https://context7.com/vercel/next.js/llms.txt?topic=cache +- **FFmpeg compression**: https://context7.com/websites/ffmpeg_doxygen_8_0/llms.txt?topic=compress + +## Official llms.txt Locations (FALLBACK) + +Use these only if context7.com returns 404: + +### JavaScript/TypeScript Frameworks + +- **Astro**: https://docs.astro.build/llms.txt +- **Next.js**: https://nextjs.org/llms.txt +- **Remix**: https://remix.run/llms.txt +- **SvelteKit**: https://kit.svelte.dev/llms.txt +- **Nuxt**: https://nuxt.com/llms.txt + +### Frontend Libraries + +- **React**: https://react.dev/llms.txt +- **Vue**: https://vuejs.org/llms.txt +- **Svelte**: https://svelte.dev/llms.txt + +### Backend/Full-stack + +- **Hono**: https://hono.dev/llms.txt +- **Fastify**: https://fastify.dev/llms.txt +- **tRPC**: https://trpc.io/llms.txt + +### Build Tools + +- **Vite**: https://vitejs.dev/llms.txt +- **Turbopack**: https://turbo.build/llms.txt + +### Databases/ORMs + +- **Prisma**: https://prisma.io/llms.txt +- **Drizzle**: https://orm.drizzle.team/llms.txt + +## Repository Patterns + +### GitHub (Most Common) + +**URL patterns:** +``` +https://github.com/[org]/[repo] +https://github.com/[user]/[repo] +``` + +**Common organization names:** +- Company name: `github.com/vercel/next.js` +- Project name: `github.com/remix-run/remix` +- Community: `github.com/facebook/react` + +**Documentation locations in repositories:** +``` +/docs/ +/documentation/ +/website/docs/ +/packages/docs/ +README.md +CONTRIBUTING.md +/examples/ +``` + +### GitLab + +**URL pattern:** +``` +https://gitlab.com/[org]/[repo] +``` + +### Bitbucket (Less Common) + +**URL pattern:** +``` +https://bitbucket.org/[org]/[repo] +``` + +## Package Registries + +### npm (JavaScript/TypeScript) + +**URL**: `https://npmjs.com/package/[name]` + +**Available info:** +- Description +- Homepage link +- Repository link +- Version history +- Dependencies + +**Useful for:** +- Finding official links +- Version information +- Package metadata + +### PyPI (Python) + +**URL**: `https://pypi.org/project/[name]` + +**Available info:** +- Description +- Documentation link +- Homepage +- Repository link +- Release history + +**Useful for:** +- Python package documentation +- Official links +- Version compatibility + +### RubyGems (Ruby) + +**URL**: `https://rubygems.org/gems/[name]` + +**Available info:** +- Description +- Homepage +- Documentation +- Source code link +- Dependencies + +**Useful for:** +- Ruby gem documentation +- Version information + +### Cargo (Rust) + +**URL**: `https://crates.io/crates/[name]` + +**Available info:** +- Description +- docs.rs link (auto-generated docs) +- Repository +- Version history + +**Useful for:** +- Rust crate documentation +- Auto-generated API docs +- Repository link + +### Maven Central (Java) + +**URL**: `https://search.maven.org/artifact/[group]/[artifact]` + +**Available info:** +- Versions +- Dependencies +- Repository link +- License + +**Useful for:** +- Java library information +- Dependency management + +## Documentation Hosting Platforms + +### Read the Docs + +**URL patterns:** +``` +https://[project].readthedocs.io +https://readthedocs.org/projects/[project] +``` + +**Features:** +- Version switching +- Multiple formats (HTML, PDF, ePub) +- Search functionality +- Often auto-generated from reStructuredText/Markdown + +### GitBook + +**URL patterns:** +``` +https://[org].gitbook.io/[project] +https://docs.[domain].com (often GitBook-powered) +``` + +**Features:** +- Clean, modern interface +- Good navigation +- Often manually curated +- May require API key for programmatic access + +### Docusaurus + +**URL patterns:** +``` +https://[project].io +https://docs.[project].com +``` + +**Common in:** +- React ecosystem +- Meta/Facebook projects +- Modern open-source projects + +**Features:** +- React-based +- Fast, static site +- Version management +- Good search + +### MkDocs + +**URL patterns:** +``` +https://[user].github.io/[project] +https://[custom-domain].com +``` + +**Features:** +- Python ecosystem +- Static site from Markdown +- Often on GitHub Pages +- Material theme popular + +### VitePress + +**URL patterns:** +``` +https://[project].dev +https://docs.[project].com +``` + +**Common in:** +- Vue ecosystem +- Modern projects +- Vite-based projects + +**Features:** +- Vue-powered +- Very fast +- Clean design +- Good DX + +## Documentation Search Patterns + +### Finding llms.txt + +**ALWAYS try context7.com first:** + +For GitHub repositories: +``` +https://context7.com/{org}/{repo}/llms.txt +``` + +For websites: +``` +https://context7.com/websites/{normalized-path}/llms.txt +``` + +With topic filter: +``` +https://context7.com/{path}/llms.txt?topic={query} +``` + +**Fallback: Traditional search if context7.com returns 404:** +``` +"[library] llms.txt site:[known-domain]" +``` + +**Alternative domains to try:** +``` +site:docs.[library].com +site:[library].dev +site:[library].io +site:[library].org +``` + +### Finding Official Repository + +**Search pattern:** +``` +"[library] official github repository" +"[library] source code github" +``` + +**Verification checklist:** +- Check organization/user is official +- Verify star count (popular libraries have many) +- Check last commit date (active maintenance) +- Look for official links in README + +### Finding Official Documentation + +**Search patterns:** +``` +"[library] official documentation" +"[library] docs site:official-domain" +"[library] API reference" +``` + +**Domain patterns:** +``` +docs.[library].com +[library].dev/docs +docs.[library].io +[library].readthedocs.io +``` + +## Common Documentation Structures + +### Typical Section Names + +**Getting started:** +- Getting Started +- Quick Start +- Introduction +- Installation +- Setup + +**Core concepts:** +- Core Concepts +- Fundamentals +- Basics +- Key Concepts +- Architecture + +**Guides:** +- Guides +- How-To Guides +- Tutorials +- Examples +- Recipes + +**Reference:** +- API Reference +- API Documentation +- Reference +- API +- CLI Reference + +**Advanced:** +- Advanced +- Advanced Topics +- Deep Dives +- Internals +- Performance + +### Common File Names + +``` +README.md +GETTING_STARTED.md +INSTALLATION.md +CONTRIBUTING.md +CHANGELOG.md +API.md +TUTORIAL.md +EXAMPLES.md +FAQ.md +``` + +## Framework-Specific Patterns + +### React Ecosystem + +**Common patterns:** +``` +- Uses Docusaurus +- Documentation at [project].dev or docs.[project].com +- Often has interactive examples +- CodeSandbox/StackBlitz embeds +``` + +### Vue Ecosystem + +**Common patterns:** +``` +- Uses VitePress +- Documentation at [project].vuejs.org +- Bilingual (English/Chinese) +- API reference auto-generated +``` + +### Python Ecosystem + +**Common patterns:** +``` +- Read the Docs hosting +- Sphinx-generated +- reStructuredText format +- [project].readthedocs.io +``` + +### Rust Ecosystem + +**Common patterns:** +``` +- docs.rs for API docs +- Book format for guides ([project].rs/book) +- Markdown in repository +- Well-structured examples/ +``` + +## Quick Lookup Table + +| Ecosystem | Registry | Docs Pattern | Common Host | +|-----------|----------|--------------|-------------| +| JavaScript/TS | npmjs.com | [name].dev | Docusaurus, VitePress | +| Python | pypi.org | readthedocs.io | Read the Docs | +| Rust | crates.io | docs.rs | docs.rs | +| Ruby | rubygems.org | rubydoc.info | RDoc | +| Go | pkg.go.dev | pkg.go.dev | pkg.go.dev | +| PHP | packagist.org | [name].org | Various | +| Java | maven.org | javadoc | Maven Central | diff --git a/skills/docs-seeker/references/error-handling.md b/skills/docs-seeker/references/error-handling.md new file mode 100644 index 0000000..9037b0d --- /dev/null +++ b/skills/docs-seeker/references/error-handling.md @@ -0,0 +1,621 @@ +# Error Handling Guide + +Comprehensive troubleshooting and error resolution strategies for documentation discovery. + +## context7.com Not Accessible + +### Symptoms + +- 404 error (library not indexed) +- Connection timeout +- Server error (500) +- Empty response + +### Troubleshooting Steps + +**1. Verify URL pattern:** + +For GitHub repos: +``` +✓ Correct: https://context7.com/vercel/next.js/llms.txt +✗ Wrong: https://context7.com/nextjs/llms.txt +``` + +For websites: +``` +✓ Correct: https://context7.com/websites/imgix/llms.txt +✗ Wrong: https://context7.com/docs.imgix.com/llms.txt +``` + +**2. Try official llms.txt as fallback:** +``` +https://docs.[library].com/llms.txt +https://[library].dev/llms.txt +https://[library].io/llms.txt +``` + +**3. Search for llms.txt if still not found:** +``` +WebSearch: "[library] llms.txt" +WebSearch: "[library] documentation AI format" +``` + +**4. Fall back to repository analysis:** +- If no llms.txt available anywhere +- Note in report: "llms.txt not available, used repository analysis" + +### Common Causes + +- Library not yet indexed by context7.com +- Very new or obscure library +- Private repository +- context7.com temporary outage + +### Example Resolution + +``` +Problem: https://context7.com/org/new-lib/llms.txt returns 404 + +Steps: +1. Check official site: https://new-lib.dev/llms.txt ✗ Not found +2. WebSearch for llms.txt ✗ Not found +3. Fall back to repository: https://github.com/org/new-lib ✓ Found +4. Use Repomix for documentation extraction +5. Note in report: "No llms.txt available, analyzed repository directly" +``` + +## llms.txt Not Accessible (Official Sites) + +### Symptoms + +- 404 error +- Connection timeout +- Access denied (403) +- Empty response + +### Troubleshooting Steps + +**1. ALWAYS try context7.com first:** +``` +https://context7.com/{org}/{repo}/llms.txt +``` + +**2. Try alternative official domains:** +``` +https://[name].dev/llms.txt +https://[name].io/llms.txt +https://[name].com/llms.txt +https://docs.[name].com/llms.txt +https://www.[name].com/llms.txt +``` + +**3. Check for redirects:** +- Old domain → new domain +- Non-HTTPS → HTTPS +- www → non-www or vice versa +- Root → /docs subdirectory + +**4. Search for llms.txt mention:** +``` +WebSearch: "[library] llms.txt" +WebSearch: "[library] documentation AI format" +``` + +**5. Check documentation announcements:** +- Blog posts about llms.txt +- GitHub discussions +- Recent release notes + +**6. If all fail:** +- Fall back to repository analysis (Phase 3) +- Note in report: "llms.txt not available" + +### Common Causes + +- Documentation recently moved/redesigned +- llms.txt not yet implemented +- Domain configuration issues +- Rate limiting or IP blocking +- Firewall/security restrictions + +### Example Resolution + +``` +Problem: https://example.dev/llms.txt returns 404 + +Steps: +1. Try: https://docs.example.dev/llms.txt ✓ Works! +2. Note: Documentation moved to docs subdomain +3. Proceed with Phase 2 using correct URL +``` + +## Repository Not Found + +### Symptoms + +- GitHub 404 error +- No official repository found +- Repository is private/requires auth +- Multiple competing repositories + +### Troubleshooting Steps + +**1. Search official website:** +``` +WebSearch: "[library] official website" +``` + +**2. Check package registries:** +``` +WebSearch: "[library] npm" +WebSearch: "[library] pypi" +WebSearch: "[library] crates.io" +``` + +**3. Look for organization GitHub:** +``` +WebSearch: "[company] github organization" +WebSearch: "[library] github org:[known-org]" +``` + +**4. Check for mirrors or forks:** +``` +WebSearch: "[library] github mirror" +WebSearch: "[library] source code" +``` + +**5. Verify through package manager:** +```bash +# npm example +npm info [package-name] repository + +# pip example +pip show [package-name] +``` + +**6. If all fail:** +- Use Researcher agents (Phase 4) +- Note: "No public repository available" + +### Common Causes + +- Proprietary/closed-source software +- Documentation separate from code repository +- Company uses internal hosting (GitLab, Bitbucket, self-hosted) +- Project discontinued or archived +- Repository renamed/moved + +### Verification Checklist + +When you find a repository, verify: +- [ ] Organization/user matches official entity +- [ ] Star count appropriate for library popularity +- [ ] Recent commits (active maintenance) +- [ ] README mentions official status +- [ ] Links back to official website +- [ ] License matches expectations + +## Repomix Failures + +### Symptoms + +- Out of memory error +- Command hangs indefinitely +- Output file empty or corrupted +- Permission errors +- Network timeout during clone + +### Troubleshooting Steps + +**1. Check repository size:** +```bash +# Clone and check size +git clone [url] /tmp/test-repo +du -sh /tmp/test-repo + +# If >500MB, use focused approach +``` + +**2. Focus on documentation only:** +```bash +repomix --include "docs/**,README.md,*.md" --output docs.xml +``` + +**3. Exclude large files:** +```bash +repomix --exclude "*.png,*.jpg,*.pdf,*.zip,dist/**,build/**,node_modules/**" --output repomix-output.xml +``` + +**4. Use shallow clone:** +```bash +git clone --depth 1 [url] /tmp/docs-analysis +cd /tmp/docs-analysis +repomix --output repomix-output.xml +``` + +**5. Alternative: Explorer agents** +``` +If Repomix fails completely: +1. Read README.md directly +2. List /docs directory structure +3. Launch Explorer agents for key files +4. Read specific documentation files +``` + +**6. Check system resources:** +```bash +# Check disk space +df -h /tmp + +# Check available memory +free -h + +# Kill if hung +pkill -9 repomix +``` + +### Common Causes + +- Repository too large (>1GB) +- Many binary files (images, videos) +- Large commit history +- Insufficient disk space +- Memory constraints +- Slow network connection +- Repository has submodules + +### Size Guidelines + +| Repo Size | Strategy | +|-----------|----------| +| <50MB | Full Repomix | +| 50-200MB | Exclude binaries | +| 200-500MB | Focus on /docs | +| 500MB-1GB | Shallow clone + focus | +| >1GB | Explorer agents only | + +## Multiple Conflicting Sources + +### Symptoms + +- Different installation instructions +- Conflicting API signatures +- Contradictory recommendations +- Version mismatches +- Breaking changes not documented + +### Resolution Steps + +**1. Check version of each source:** +``` +- Note documentation version number +- Check last-updated date +- Check URL for version indicator (v1/, v2/) +- Look for version selector on page +``` + +**2. Prioritize sources:** +``` +Priority order: +1. Official docs (latest version) +2. Official docs (specified version) +3. Package registry (verified) +4. Official repository README +5. Community tutorials (recent) +6. Stack Overflow (recent, high votes) +7. Blog posts (date-verified) +``` + +**3. Present both with context:** +```markdown +## Installation (v1.x - Legacy) +[old method] +Source: [link] (Last updated: [date]) + +## Installation (v2.x - Current) +[new method] +Source: [link] (Last updated: [date]) + +⚠️ Note: v2.x is recommended for new projects. +Migration guide: [link] +``` + +**4. Cross-reference:** +- Check if conflict is intentional (breaking change) +- Look for migration guides +- Check changelog/release notes +- Verify in GitHub issues/discussions + +**5. Document discrepancy:** +```markdown +## ⚠️ Conflicting Information Found + +**Source 1** (official docs): Method A +**Source 2** (repository): Method B + +**Analysis**: Source 1 reflects v2.x API. Source 2 README +not yet updated. Confirmed via changelog [link]. + +**Recommendation**: Use Method A (official docs). +``` + +### Version Identification + +**Check these locations:** +``` +- URL path: /docs/v2/... +- Page header/footer +- Version selector dropdown +- Git branch/tag +- Package.json or equivalent +- CHANGELOG.md date correlation +``` + +## Rate Limiting + +### Symptoms + +- 429 Too Many Requests +- 403 Forbidden (temporary) +- Slow responses +- Connection refused +- "Rate limit exceeded" message + +### Solutions + +**1. Add delays between requests:** +```bash +# Add 2-second delay +sleep 2 +``` + +**2. Use alternative sources:** +``` +Priority fallback chain: +GitHub → Official docs → Package registry → Repository → Archive +``` + +**3. Batch operations:** +``` +Instead of: +- WebFetch URL 1 +- WebFetch URL 2 +- WebFetch URL 3 + +Use: +- Launch 3 Explorer agents (single batch) +``` + +**4. Cache aggressively:** +``` +- Reuse fetched content within session +- Don't re-fetch same URLs +- Store repomix output for reuse +- Note fetch time, reuse if <1 hour old +``` + +**5. Check rate limit headers:** +``` +If available: +- X-RateLimit-Remaining +- X-RateLimit-Reset +- Retry-After +``` + +**6. Respect robots.txt:** +```bash +# Check before aggressive crawling +curl https://example.com/robots.txt +``` + +### Rate Limit Recovery + +**GitHub API (if applicable):** +``` +- Anonymous: 60 requests/hour +- Authenticated: 5000 requests/hour +- Wait period: 1 hour from first request +``` + +**General approach:** +``` +1. Detect rate limit (429 or slow responses) +2. Switch to alternative source immediately +3. Don't retry same endpoint repeatedly +4. Note in report: "Rate limit encountered, used [alternative]" +``` + +## Network Timeouts + +### Symptoms + +- Request hangs indefinitely +- Connection timeout error +- No response received +- Partial content received + +### Solutions + +**1. Set explicit timeouts:** +``` +WebSearch: 30 seconds max +WebFetch: 60 seconds max +Repository clone: 5 minutes max +Repomix processing: 10 minutes max +``` + +**2. Retry with timeout:** +``` +1st attempt: 60 seconds +2nd attempt: 90 seconds (if needed) +3rd attempt: Switch to alternative method +``` + +**3. Check network connectivity:** +```bash +# Test basic connectivity +ping -c 3 8.8.8.8 + +# Test DNS resolution +nslookup docs.example.com + +# Test specific host +curl -I https://docs.example.com +``` + +**4. Use alternative endpoints:** +``` +If main site times out: +- Try CDN version +- Try regional mirror +- Try cached version (Google Cache, Archive.org) +``` + +**5. Fall back gracefully:** +``` +Main docs timeout → Repository → Package registry → Research +``` + +## Incomplete Documentation + +### Symptoms + +- Documentation stub pages +- "Coming soon" sections +- Broken links (404) +- Missing API reference +- Outdated examples + +### Handling Strategy + +**1. Identify gaps:** +```markdown +## Documentation Status + +✅ Available: +- Installation guide +- Basic usage examples + +⚠️ Incomplete: +- Advanced features (stub page) +- API reference (404 links) + +❌ Missing: +- Migration guide +- Performance optimization +``` + +**2. Supplement from repository:** +``` +- Check /examples directory +- Read test files for usage +- Analyze TypeScript definitions +- Check CHANGELOG for features +``` + +**3. Use community sources:** +``` +- Recent Stack Overflow answers +- GitHub discussions +- Blog posts from maintainers +- Video tutorials +``` + +**4. Note limitations clearly:** +```markdown +⚠️ **Documentation Limitations** + +Official docs incomplete (as of [date]). +The following information inferred from: +- Repository examples +- TypeScript definitions +- Community discussions + +May not reflect official recommendations. +``` + +## Authentication/Access Issues + +### Symptoms + +- Private repository +- Login required +- Organization-only access +- Documentation behind paywall + +### Solutions + +**1. For private repositories:** +``` +- Note: "Repository is private" +- Check for public mirror +- Look for public documentation site +- Search package registry for info +``` + +**2. For paywalled docs:** +``` +- Check for free tier/trial +- Look for open-source alternative +- Search for community mirrors +- Use package registry info instead +``` + +**3. Document access limitation:** +```markdown +## ⚠️ Access Limitation + +Official repository is private. This report based on: +- Public documentation site: [url] +- Package registry info: [url] +- Community resources: [urls] + +May not include internal implementation details. +``` + +## Error Handling Best Practices + +### General Principles + +1. **Fail fast**: Don't retry same method repeatedly +2. **Fall back**: Have alternative strategies ready +3. **Document**: Note what failed and why +4. **Inform user**: Clear about limitations +5. **Partial success**: Deliver what you can find + +### Error Reporting Template + +```markdown +## ⚠️ Discovery Issues Encountered + +**Primary method**: [method] - [reason for failure] +**Fallback used**: [alternative method] +**Information completeness**: [percentage or description] + +**What was found**: +- [list available information] + +**What is missing**: +- [list gaps] + +**Recommended action**: +- [how user can get missing info] +``` + +### Recovery Decision Tree + +``` +Error encountered + ↓ +Is there an obvious alternative? + YES → Try alternative immediately + NO → Continue below + ↓ +Have we tried all primary methods? + NO → Try next method in sequence + YES → Continue below + ↓ +Is partial information useful? + YES → Deliver partial results with notes + NO → Inform user, request guidance +``` diff --git a/skills/docs-seeker/references/limitations.md b/skills/docs-seeker/references/limitations.md new file mode 100644 index 0000000..b17b73d --- /dev/null +++ b/skills/docs-seeker/references/limitations.md @@ -0,0 +1,821 @@ +# Limitations & Success Criteria + +Understanding boundaries and measuring effectiveness of documentation discovery. + +## context7.com Limitations + +### Not All Libraries Indexed + +**Limitation:** +- context7.com doesn't index every repository/website +- Very new libraries may not be available yet +- Private repositories not accessible +- Some niche libraries missing + +**Impact:** +- Need fallback to official llms.txt or repository analysis +- May add 10-20 seconds to discovery time +- Requires manual search for obscure libraries + +**Workarounds:** +``` +1. Try context7.com first (always) +2. If 404, fall back to official llms.txt search +3. If still not found, use repository analysis +4. Note in report which method was used +``` + +**Example:** +``` +Tried: https://context7.com/org/new-lib/llms.txt → 404 +Fallback: WebSearch for "new-lib llms.txt" → Found +Used: Official llms.txt from website +``` + +### Topic Filtering Accuracy + +**Limitation:** +- ?topic= parameter relies on keyword matching +- May miss relevant content with different terminology +- May include tangentially related content +- Quality depends on context7 indexing + +**Impact:** +- Occasionally need to review base llms.txt without topic filter +- May miss some relevant documentation + +**Workarounds:** +``` +- Try multiple topic keywords +- Fall back to full llms.txt if topic search insufficient +- Use broader terms for better coverage +``` + +## Cannot Handle + +### Password-Protected Documentation + +**Limitation:** +- No access to authentication-required content +- Cannot log in to platforms +- No credential management +- Cannot access organization-internal docs + +**Impact:** +- Enterprise documentation inaccessible +- Premium content unavailable +- Private beta docs unreachable +- Internal wikis not readable + +**Workarounds:** +``` +- Ask user for public alternatives +- Search for public subset of docs +- Use publicly available README/marketing +- Check if trial/demo access available +- Note limitation in report +``` + +**Report template:** +```markdown +⚠️ **Access Limitation** + +Documentation requires authentication. + +**What we can access**: +- Public README: [url] +- Package registry info: [url] +- Marketing site: [url] + +**Cannot access**: +- Full documentation (requires login) +- Internal guides +- Premium content + +**Recommendation**: Contact vendor for access or check if public docs available. +``` + +### Rate-Limited APIs + +**Limitation:** +- No API credentials for authenticated access +- Subject to anonymous rate limits +- Cannot request increased limits +- No retry with authentication + +**Impact:** +- Limited requests per hour (e.g., GitHub: 60/hour anonymous) +- May hit limits during comprehensive search +- Slower fallback required +- Incomplete coverage possible + +**Workarounds:** +``` +- Add delays between requests +- Use alternative sources (cached, mirrors) +- Prioritize critical pages +- Use Researcher agents instead of API +- Switch to repository analysis +``` + +**Detection:** +``` +Symptoms: +- 429 Too Many Requests +- X-RateLimit-Remaining: 0 +- Slow or refused connections + +Response: +- Immediately switch to alternative method +- Don't retry same endpoint +- Note in report which method used +``` + +### Real-Time Documentation + +**Limitation:** +- Uses snapshot at time of access +- Cannot monitor for updates +- No real-time synchronization +- May miss very recent changes + +**Impact:** +- Documentation updated minutes ago may not be reflected +- Breaking changes announced today might be missed +- Latest release notes may not be current +- Version just released may not be documented + +**Workarounds:** +``` +- Note access date in report +- Recommend user verify if critical +- Check last-modified headers +- Compare with release dates +- Suggest official site for latest +``` + +**Report template:** +```markdown +ℹ️ **Snapshot Information** + +Documentation retrieved: 2025-10-26 14:30 UTC + +**Last-Modified** (if available): +- Main docs: 2025-10-24 +- API reference: 2025-10-22 + +**Note**: For real-time updates, check official site: [url] +``` + +### Interactive Documentation + +**Limitation:** +- Cannot run interactive examples +- Cannot execute code playgrounds +- No ability to test API calls +- Cannot verify functionality + +**Impact:** +- Cannot confirm examples work +- Cannot test edge cases +- Cannot validate API responses +- Cannot verify performance claims + +**Workarounds:** +``` +- Provide code examples as-is +- Note: "Example provided, not tested" +- Recommend user run examples +- Link to interactive playground if available +- Include caveats about untested code +``` + +**Report template:** +```markdown +## Example Usage + +```python +# Example from official docs (not tested) +import library +result = library.do_thing() +``` + +⚠️ **Note**: Example provided from documentation but not executed. +Please test in your environment. + +**Interactive playground**: [url if available] +``` + +### Video-Only Documentation + +**Limitation:** +- Cannot process video content directly +- Limited transcript access +- Cannot extract code from video +- Cannot parse visual diagrams + +**Impact:** +- Video tutorials not usable +- YouTube courses inaccessible +- Screencasts not processable +- Visual walkthroughs unavailable + +**Workarounds:** +``` +- Search for transcript if available +- Look for accompanying blog post +- Find text-based alternative +- Check for community notes +- Use automated captions if available (low quality) +``` + +**Report template:** +```markdown +ℹ️ **Video Content Detected** + +Primary documentation is video-based: [url] + +**Alternatives found**: +- Blog post summary: [url] +- Community notes: [url] + +**Cannot extract**: +- Detailed walkthrough from video +- Visual examples +- Demonstration steps + +**Recommendation**: Watch video directly for visual content. +``` + +## May Struggle With + +### Very Large Repositories (>1GB) + +**Challenge:** +- Repomix may fail or hang +- Clone takes very long time +- Processing exceeds memory limits +- Output file too large to read + +**Success rate:** ~30% for >1GB repos + +**Mitigation:** +``` +1. Try shallow clone: git clone --depth 1 +2. Focus on docs only: repomix --include "docs/**" +3. Exclude binaries: --exclude "*.png,*.jpg,dist/**" +4. If fails: Use Explorer agents on specific files +5. Note limitation in report +``` + +**When to skip:** +``` +Repository size indicators: +- Git clone shows >1GB download +- Contains large binaries (ml models, datasets) +- Has extensive history (>10k commits) +- Many multimedia files + +→ Skip Repomix, use targeted exploration +``` + +### Documentation in Images/PDFs + +**Challenge:** +- Cannot reliably extract text from images +- PDF parsing limited +- Formatting often lost +- Code snippets may be corrupted + +**Success rate:** ~50% quality for PDFs, ~10% for images + +**Mitigation:** +``` +1. Search for text alternative +2. Try OCR if critical (low quality) +3. Provide image URL instead +4. Note content not extractable +5. Recommend manual review +``` + +**Report template:** +```markdown +⚠️ **Image-Based Documentation** + +Primary documentation in PDF/images: [url] + +**Extraction quality**: Limited +**Recommendation**: Download and review manually + +**Text alternatives found**: +- [any alternatives] +``` + +### Non-English Documentation + +**Challenge:** +- No automatic translation +- May miss context/nuance +- Technical terms may not translate well +- Examples may be language-specific + +**Success rate:** Variable (depends on user needs) + +**Mitigation:** +``` +1. Note language in report +2. Offer key section translation if user requests +3. Search for English version +4. Check if bilingual docs exist +5. Provide original with language note +``` + +**Report template:** +```markdown +ℹ️ **Language Notice** + +Primary documentation in: Japanese + +**English availability**: +- Partial translation: [url] +- Community translation: [url] +- No official English version found + +**Recommendation**: Use translation tool or request community help. +``` + +### Scattered Documentation + +**Challenge:** +- Multiple sites/repositories +- Inconsistent structure +- Conflicting information +- No central source + +**Success rate:** ~60% coverage + +**Mitigation:** +``` +1. Use Researcher agents +2. Prioritize official sources +3. Cross-reference findings +4. Note conflicts clearly +5. Take longer but be thorough +``` + +**Report template:** +```markdown +ℹ️ **Fragmented Documentation** + +Information found across multiple sources: + +**Official** (incomplete): +- Website: [url] +- Package registry: [url] + +**Community** (supplementary): +- Stack Overflow: [url] +- Tutorial: [url] + +**Note**: No centralized documentation. Information aggregated from +multiple sources. Conflicts resolved by prioritizing official sources. +``` + +### Deprecated/Legacy Libraries + +**Challenge:** +- Documentation removed or archived +- Only old versions available +- Outdated information +- No current maintenance + +**Success rate:** ~40% for fully deprecated libraries + +**Mitigation:** +``` +1. Use Internet Archive (Wayback Machine) +2. Search GitHub repository history +3. Check package registry for old README +4. Look for fork with docs +5. Note legacy status clearly +``` + +**Report template:** +```markdown +⚠️ **Legacy Library** + +**Status**: Deprecated as of [date] +**Last update**: [date] + +**Documentation sources**: +- Archived docs (via Wayback): [url] +- Repository (last commit [date]): [url] + +**Recommendation**: Consider modern alternative: [suggestion] + +**Migration path**: [if available] +``` + +## Success Criteria + +### 1. Finds Relevant Information + +**Measured by:** +- [ ] Answers user's specific question +- [ ] Covers requested topics +- [ ] Appropriate depth/detail +- [ ] Includes practical examples +- [ ] Links to additional resources + +**Quality levels:** + +**Excellent (100%):** +``` +- All requested topics covered +- Examples for each major concept +- Clear, comprehensive information +- Official source, current version +- No gaps or limitations +``` + +**Good (80-99%):** +``` +- Most requested topics covered +- Examples for core concepts +- Information mostly complete +- Official source, some gaps noted +- Minor limitations +``` + +**Acceptable (60-79%):** +``` +- Core topics covered +- Some examples present +- Information somewhat complete +- Mix of official/community sources +- Some gaps noted +``` + +**Poor (<60%):** +``` +- Only partial coverage +- Few or no examples +- Significant gaps +- Mostly unofficial sources +- Many limitations +``` + +### 2. Uses Most Efficient Method + +**Measured by:** +- [ ] Started with llms.txt +- [ ] Used parallel agents appropriately +- [ ] Avoided unnecessary operations +- [ ] Completed in reasonable time +- [ ] Fell back efficiently when needed + +**Efficiency score:** + +**Optimal:** +``` +- Found llms.txt immediately +- Parallel agents for all URLs +- Single batch processing +- Completed in <2 minutes +- No wasted operations +``` + +**Good:** +``` +- Found llms.txt after 1-2 tries +- Mostly parallel processing +- Minimal sequential operations +- Completed in <5 minutes +- One minor inefficiency +``` + +**Acceptable:** +``` +- Fell back to repository after llms.txt search +- Mix of parallel and sequential +- Some redundant operations +- Completed in <10 minutes +- A few inefficiencies +``` + +**Poor:** +``` +- Didn't try llms.txt first +- Mostly sequential processing +- Many redundant operations +- Took >10 minutes +- Multiple inefficiencies +``` + +### 3. Completes in Reasonable Time + +**Target times:** + +| Scenario | Excellent | Good | Acceptable | Poor | +|----------|-----------|------|------------|------| +| Simple (1-5 URLs) | <1 min | 1-2 min | 2-5 min | >5 min | +| Medium (6-15 URLs) | <2 min | 2-4 min | 4-7 min | >7 min | +| Complex (16+ URLs) | <3 min | 3-6 min | 6-10 min | >10 min | +| Repository | <3 min | 3-6 min | 6-10 min | >10 min | +| Research | <5 min | 5-8 min | 8-12 min | >12 min | + +**Factors affecting time:** +- Documentation structure (well-organized vs scattered) +- Source availability (llms.txt vs research) +- Content volume (few pages vs many) +- Network conditions (fast vs slow) +- Complexity (simple vs comprehensive) + +### 4. Provides Clear Source Attribution + +**Measured by:** +- [ ] Lists all sources used +- [ ] Notes method employed +- [ ] Includes URLs/references +- [ ] Identifies official vs community +- [ ] Credits authors when relevant + +**Quality template:** + +**Excellent:** +```markdown +## Sources + +**Primary method**: llms.txt +**URL**: https://docs.library.com/llms.txt + +**Documentation retrieved**: +1. Getting Started (official): [url] +2. API Reference (official): [url] +3. Examples (official): [url] + +**Additional sources**: +- Repository: https://github.com/org/library +- Package registry: https://npmjs.com/package/library + +**Method**: Parallel exploration with 3 agents +**Date**: 2025-10-26 14:30 UTC +``` + +### 5. Identifies Version/Date + +**Measured by:** +- [ ] Documentation version noted +- [ ] Last-updated date included +- [ ] Matches user's version requirement +- [ ] Flags if version mismatch +- [ ] Notes if version unclear + +**Best practice:** +```markdown +## Version Information + +**Documentation version**: v3.2.1 +**Last updated**: 2025-10-20 +**Retrieved**: 2025-10-26 + +**User requested**: v3.x ✓ Match + +**Note**: This is the latest stable version as of retrieval date. +``` + +### 6. Notes Limitations/Gaps + +**Measured by:** +- [ ] Missing information identified +- [ ] Incomplete sections noted +- [ ] Known issues mentioned +- [ ] Alternatives suggested +- [ ] Workarounds provided + +**Good practice:** +```markdown +## ⚠️ Limitations + +**Incomplete documentation**: +- Advanced features section (stub page) +- Migration guide (404 error) + +**Not available**: +- Video tutorials mentioned but not accessible +- Interactive examples require login + +**Workarounds**: +- Advanced features: See examples in repository +- Migration: Check CHANGELOG.md for breaking changes + +**Alternatives**: +- Community tutorial: [url] +- Stack Overflow: [url] +``` + +### 7. Well-Organized Output + +**Measured by:** +- [ ] Clear structure +- [ ] Logical flow +- [ ] Easy to scan +- [ ] Actionable information +- [ ] Proper formatting + +**Structure template:** +```markdown +# Documentation for [Library] [Version] + +## Overview +[Brief description] + +## Source +[Attribution] + +## Installation +[Step-by-step] + +## Quick Start +[Minimal example] + +## Core Concepts +[Main ideas] + +## API Reference +[Key methods] + +## Examples +[Practical usage] + +## Additional Resources +[Links] + +## Limitations +[Any gaps] +``` + +## Quality Checklist + +### Before Presenting Report + +**Content quality:** +- [ ] Information is accurate +- [ ] Sources are official (or noted as unofficial) +- [ ] Version matches request +- [ ] Examples are clear +- [ ] No obvious errors + +**Completeness:** +- [ ] All key topics covered +- [ ] Installation instructions present +- [ ] Usage examples included +- [ ] Configuration documented +- [ ] Troubleshooting information available + +**Organization:** +- [ ] Logical flow +- [ ] Clear headings +- [ ] Proper code formatting +- [ ] Links working (spot check) +- [ ] Easy to scan + +**Attribution:** +- [ ] Sources listed +- [ ] Method documented +- [ ] Version identified +- [ ] Date noted +- [ ] Limitations disclosed + +**Usability:** +- [ ] Actionable information +- [ ] Copy-paste ready examples +- [ ] Next steps clear +- [ ] Resources for deep dive +- [ ] Known issues noted + +## Performance Metrics + +### Time-to-Value + +**Measures:** How quickly user gets useful information + +**Targets:** +``` +First useful info: <30 seconds +Core coverage: <2 minutes +Complete report: <5 minutes +``` + +**Tracking:** +``` +Start → llms.txt found → First URL fetched → Critical info extracted + 15s 30s 45s 60s + +User can act on info at 60s mark +(even if full report takes 5 minutes) +``` + +### Coverage Completeness + +**Measures:** Percentage of user needs met + +**Calculation:** +``` +User needs 5 topics: +- Installation ✓ +- Basic usage ✓ +- API reference ✓ +- Configuration ✓ +- Examples ✗ (not found) + +Coverage: 4/5 = 80% +``` + +**Targets:** +``` +Excellent: 90-100% +Good: 75-89% +Acceptable: 60-74% +Poor: <60% +``` + +### Source Quality + +**Measures:** Reliability of sources used + +**Scoring:** +``` +Official docs: 100 points +Official repository: 80 points +Package registry: 60 points +Recent community (verified): 40 points +Old community (unverified): 20 points +``` + +**Target:** Average >70 points + +### User Satisfaction Indicators + +**Positive signals:** +``` +- User proceeds immediately with info +- No follow-up questions needed +- User says "perfect" or "thanks" +- User marks conversation complete +``` + +**Negative signals:** +``` +- User asks same question differently +- User requests more details +- User says "incomplete" or "not what I needed" +- User abandons task +``` + +## Continuous Improvement + +### Learn from Failures + +**When documentation discovery struggles:** + +1. **Document the issue** + ``` + - What was attempted + - What failed + - Why it failed + - How it was resolved (if at all) + ``` + +2. **Identify patterns** + ``` + - Is this library-specific? + - Is this ecosystem-specific? + - Is this a general limitation? + ``` + +3. **Update strategies** + ``` + - Add workaround to playbook + - Update fallback sequence + - Note limitation in documentation + ``` + +### Measure and Optimize + +**Track these metrics:** +``` +- Average time to complete +- Coverage percentage +- Source quality score +- User satisfaction +- Failure rate by method +``` + +**Optimize based on data:** +``` +- Which method succeeds most often? +- Which ecosystems need special handling? +- Where are time bottlenecks? +- What causes most failures? +``` diff --git a/skills/docs-seeker/references/performance.md b/skills/docs-seeker/references/performance.md new file mode 100644 index 0000000..e632aaf --- /dev/null +++ b/skills/docs-seeker/references/performance.md @@ -0,0 +1,574 @@ +# Performance Optimization + +Strategies and techniques for maximizing speed and efficiency in documentation discovery. + +## Core Principles + +### 0. Use context7.com for Instant llms.txt Access + +**Fastest Approach:** + +Direct URL construction instead of searching: +``` +Traditional: WebSearch (15-30s) → WebFetch (5-10s) = 20-40s +context7.com: Direct WebFetch (5-10s) = 5-10s + +Speed improvement: 2-4x faster +``` + +**Benefits:** +- No search required (instant URL construction) +- Consistent URL patterns +- Reliable availability +- Topic filtering for targeted results + +**Examples:** +``` +GitHub repo: +https://context7.com/vercel/next.js/llms.txt +→ Instant, no search needed + +Website: +https://context7.com/websites/imgix/llms.txt +→ Instant, no search needed + +Topic-specific: +https://context7.com/shadcn-ui/ui/llms.txt?topic=date +→ Filtered results, even faster +``` + +**Performance Impact:** +``` +Without context7.com: +1. WebSearch for llms.txt: 15s +2. WebFetch llms.txt: 5s +3. Launch agents: 5s +Total: 25s + +With context7.com: +1. Direct WebFetch: 5s +2. Launch agents: 5s +Total: 10s (2.5x faster!) + +With context7.com + topic: +1. Direct WebFetch (filtered): 3s +2. Process focused results: 2s +Total: 5s (5x faster!) +``` + +### 1. Minimize Sequential Operations + +**The Problem:** + +Sequential operations add up linearly: +``` +Total Time = Op1 + Op2 + Op3 + ... + OpN +``` + +Example: +``` +Fetch URL 1: 5 seconds +Fetch URL 2: 5 seconds +Fetch URL 3: 5 seconds +Total: 15 seconds +``` + +**The Solution:** + +Parallel operations complete in max time of slowest: +``` +Total Time = max(Op1, Op2, Op3, ..., OpN) +``` + +Example: +``` +Launch 3 agents simultaneously +All complete in: ~5 seconds +Total: 5 seconds (3x faster!) +``` + +### 2. Batch Related Operations + +**Benefits:** +- Fewer context switches +- Better resource utilization +- Easier to track +- More efficient aggregation + +**Grouping Strategies:** + +**By topic:** +``` +Agent 1: All authentication-related docs +Agent 2: All database-related docs +Agent 3: All API-related docs +``` + +**By content type:** +``` +Agent 1: All tutorials +Agent 2: All reference docs +Agent 3: All examples +``` + +**By priority:** +``` +Phase 1 (critical): Getting started, installation, core concepts +Phase 2 (important): Guides, API reference, configuration +Phase 3 (optional): Advanced topics, internals, optimization +``` + +### 3. Smart Caching + +**What to cache:** +- Repomix output (expensive to generate) +- llms.txt content (static) +- Repository structure (rarely changes) +- Documentation URLs (reference list) + +**When to refresh:** +- User requests specific version +- Documentation updated (check last-modified) +- Cache older than session +- User explicitly requests fresh data + +### 4. Early Termination + +**When to stop:** +``` +✓ User's core needs met +✓ Critical information found +✓ Time limit approaching +✓ Diminishing returns (90% coverage achieved) +``` + +**How to decide:** +``` +After Phase 1 (critical docs): +- Review what was found +- Check against user request +- If 80%+ covered → deliver now +- Offer to fetch more if needed +``` + +## Performance Patterns + +### Pattern 1: Parallel Exploration + +**Scenario:** llms.txt contains 10 URLs + +**Slow approach (sequential):** +``` +Time: 10 URLs × 5 seconds = 50 seconds + +Step 1: Fetch URL 1 (5s) +Step 2: Fetch URL 2 (5s) +Step 3: Fetch URL 3 (5s) +... +Step 10: Fetch URL 10 (5s) +``` + +**Fast approach (parallel):** +``` +Time: ~5-10 seconds total + +Step 1: Launch 5 Explorer agents (simultaneous) + Agent 1: URLs 1-2 + Agent 2: URLs 3-4 + Agent 3: URLs 5-6 + Agent 4: URLs 7-8 + Agent 5: URLs 9-10 + +Step 2: Wait for all (max time: ~5-10s) +Step 3: Aggregate results +``` + +**Speedup:** 5-10x faster + +### Pattern 2: Lazy Loading + +**Scenario:** Documentation has 30+ pages + +**Slow approach (fetch everything):** +``` +Time: 30 URLs × 5 seconds ÷ 5 agents = 30 seconds + +Fetch all 30 pages upfront +User only needs 5 of them +Wasted: 25 pages × 5 seconds ÷ 5 = 25 seconds +``` + +**Fast approach (priority loading):** +``` +Time: 10 URLs × 5 seconds ÷ 5 agents = 10 seconds + +Phase 1: Fetch critical 10 pages +Review: Does this cover user's needs? +If yes: Stop here (saved 20 seconds) +If no: Fetch additional as needed +``` + +**Speedup:** Up to 3x faster for typical use cases + +### Pattern 3: Smart Fallbacks + +**Scenario:** llms.txt not found + +**Slow approach (exhaustive search):** +``` +Time: ~5 minutes + +Try: docs.library.com/llms.txt (30s timeout) +Try: library.dev/llms.txt (30s timeout) +Try: library.io/llms.txt (30s timeout) +Try: library.org/llms.txt (30s timeout) +Try: www.library.com/llms.txt (30s timeout) +Then: Fall back to repository +``` + +**Fast approach (quick fallback):** +``` +Time: ~1 minute + +Try: docs.library.com/llms.txt (15s) +Try: library.dev/llms.txt (15s) +Not found → Immediately try repository (30s) +``` + +**Speedup:** 5x faster + +### Pattern 4: Incremental Results + +**Scenario:** Large documentation set + +**Slow approach (all-or-nothing):** +``` +Time: 5 minutes until first result + +Fetch all documentation +Aggregate everything +Present complete report +User waits 5 minutes +``` + +**Fast approach (streaming):** +``` +Time: 30 seconds to first result + +Phase 1: Fetch critical docs (30s) +Present: Initial findings +Phase 2: Fetch important docs (60s) +Update: Additional findings +Phase 3: Fetch supplementary (90s) +Final: Complete report +``` + +**Benefit:** User gets value immediately, can stop early if satisfied + +## Optimization Techniques + +### Technique 1: Workload Balancing + +**Problem:** Uneven distribution causes bottlenecks + +``` +Bad distribution: +Agent 1: 1 URL (small) → finishes in 5s +Agent 2: 10 URLs (large) → finishes in 50s +Total: 50s (bottlenecked by Agent 2) +``` + +**Solution:** Balance by estimated size + +``` +Good distribution: +Agent 1: 3 URLs (medium pages) → ~15s +Agent 2: 3 URLs (medium pages) → ~15s +Agent 3: 3 URLs (medium pages) → ~15s +Agent 4: 1 URL (large page) → ~15s +Total: ~15s (balanced) +``` + +### Technique 2: Request Coalescing + +**Problem:** Redundant requests slow things down + +``` +Bad: +Agent 1: Fetch README.md +Agent 2: Fetch README.md (duplicate!) +Agent 3: Fetch README.md (duplicate!) +Wasted: 2 redundant fetches +``` + +**Solution:** Deduplicate before fetching + +``` +Good: +Pre-processing: Identify unique URLs +Agent 1: Fetch README.md (once) +Agent 2: Fetch INSTALL.md +Agent 3: Fetch API.md +Share: README.md content across agents if needed +``` + +### Technique 3: Timeout Tuning + +**Problem:** Default timeouts too conservative + +``` +Slow: +WebFetch timeout: 120s (too long for fast sites) +If site is down: Wait 120s before failing +``` + +**Solution:** Adaptive timeouts + +``` +Fast: +Known fast sites (official docs): 30s timeout +Unknown sites: 60s timeout +Large repos: 120s timeout +If timeout hit: Immediately try alternative +``` + +### Technique 4: Selective Fetching + +**Problem:** Fetching irrelevant content + +``` +Wasteful: +Fetch: Installation guide ✓ (needed) +Fetch: API reference ✓ (needed) +Fetch: Internal architecture ✗ (not needed for basic usage) +Fetch: Contributing guide ✗ (not needed) +Fetch: Changelog ✗ (not needed) +``` + +**Solution:** Filter by user needs + +``` +Efficient: +User need: "How to get started" +Fetch only: Installation, basic usage, examples +Skip: Advanced topics, internals, contribution +Speedup: 50% less fetching +``` + +## Performance Benchmarks + +### Target Times + +| Scenario | Target Time | Acceptable | Too Slow | +|----------|-------------|------------|----------| +| Single URL | <10s | 10-20s | >20s | +| llms.txt (5 URLs) | <30s | 30-60s | >60s | +| llms.txt (15 URLs) | <60s | 60-120s | >120s | +| Repository analysis | <2min | 2-5min | >5min | +| Research fallback | <3min | 3-7min | >7min | + +### Real-World Examples + +**Fast case (Next.js with llms.txt):** +``` +00:00 - Start +00:05 - Found llms.txt +00:10 - Fetched content (12 URLs) +00:15 - Launched 4 agents +00:45 - All agents complete +00:55 - Report ready +Total: 55 seconds ✓ +``` + +**Medium case (Repository without llms.txt):** +``` +00:00 - Start +00:15 - llms.txt not found +00:20 - Found repository +00:30 - Cloned repository +02:00 - Repomix complete +02:30 - Analyzed output +02:45 - Report ready +Total: 2m 45s ✓ +``` + +**Slow case (Scattered documentation):** +``` +00:00 - Start +00:30 - llms.txt not found +00:45 - Repository not found +01:00 - Launched 4 Researcher agents +05:00 - All research complete +06:00 - Aggregated findings +06:30 - Report ready +Total: 6m 30s (acceptable for research) +``` + +## Common Performance Issues + +### Issue 1: Too Many Agents + +**Symptom:** Slower than sequential + +``` +Problem: +Launched 15 agents for 15 URLs +Overhead: Agent initialization, coordination +Result: Slower than 5 agents with 3 URLs each +``` + +**Solution:** +``` +Max 7 agents per batch +Group URLs sensibly +Use phases for large sets +``` + +### Issue 2: Blocking Operations + +**Symptom:** Agents waiting unnecessarily + +``` +Problem: +Agent 1: Fetch URL, wait for Agent 2 +Agent 2: Fetch URL, wait for Agent 3 +Agent 3: Fetch URL +Result: Sequential instead of parallel +``` + +**Solution:** +``` +Launch all agents independently +No dependencies between agents +Aggregate after all complete +``` + +### Issue 3: Redundant Fetching + +**Symptom:** Same content fetched multiple times + +``` +Problem: +Phase 1: Fetch installation guide +Phase 2: Fetch installation guide again +Result: Wasted time +``` + +**Solution:** +``` +Cache fetched content +Check cache before fetching +Reuse within session +``` + +### Issue 4: Late Bailout + +**Symptom:** Continuing when should stop + +``` +Problem: +Found 90% of needed info after 1 minute +Spent 4 more minutes on remaining 10% +Result: 5x time for marginal gain +``` + +**Solution:** +``` +Check progress after critical phase +If 80%+ covered → offer to stop +Only continue if user wants comprehensive +``` + +## Performance Monitoring + +### Key Metrics + +**Track these times:** +``` +- llms.txt discovery: Target <30s +- Repository clone: Target <60s +- Repomix processing: Target <2min +- Agent exploration: Target <60s +- Total time: Target <3min for typical case +``` + +### Performance Report Template + +```markdown +## Performance Summary + +**Total time**: 1m 25s +**Method**: llms.txt + parallel exploration + +**Breakdown**: +- Discovery: 15s (llms.txt search & fetch) +- Exploration: 50s (4 agents, 12 URLs) +- Aggregation: 20s (synthesis & formatting) + +**Efficiency**: 8.5x faster than sequential +(12 URLs × 5s = 60s sequential, actual: 50s parallel) +``` + +### When to Optimize Further + +Optimize if: +- [ ] Total time >2x target +- [ ] User explicitly requests "fast" +- [ ] Repeated similar queries (cache benefit) +- [ ] Large documentation set (>20 URLs) + +Don't over-optimize if: +- [ ] Already meeting targets +- [ ] One-time query +- [ ] User values completeness over speed +- [ ] Research requires thoroughness + +## Quick Optimization Checklist + +### Before Starting + +- [ ] Check if content already cached +- [ ] Identify fastest method for this case +- [ ] Plan for parallel execution +- [ ] Set appropriate timeouts + +### During Execution + +- [ ] Launch agents in parallel (not sequential) +- [ ] Use single message for multiple agents +- [ ] Monitor for bottlenecks +- [ ] Be ready to terminate early + +### After First Phase + +- [ ] Assess coverage achieved +- [ ] Determine if user needs met +- [ ] Decide: continue or deliver now +- [ ] Cache results for potential reuse + +### Optimization Decision Tree + +``` +Need documentation? + ↓ +Check cache + ↓ +HIT → Use cached (0s) ✓ +MISS → Continue + ↓ +llms.txt available? + ↓ +YES → Parallel agents (30-60s) ✓ +NO → Continue + ↓ +Repository available? + ↓ +YES → Repomix (2-5min) +NO → Research (3-7min) + ↓ +After Phase 1: +80%+ coverage? + ↓ +YES → Deliver now (save time) ✓ +NO → Continue to Phase 2 +``` diff --git a/skills/docs-seeker/references/tool-selection.md b/skills/docs-seeker/references/tool-selection.md new file mode 100644 index 0000000..a76a5f9 --- /dev/null +++ b/skills/docs-seeker/references/tool-selection.md @@ -0,0 +1,262 @@ +# Tool Selection Guide + +Complete reference for choosing and using the right tools for documentation discovery. + +## context7.com (PRIORITY) + +**Use FIRST for all llms.txt lookups** + +**Patterns:** + +GitHub repositories: +``` +Pattern: https://context7.com/{org}/{repo}/llms.txt +Examples: +- https://github.com/vercel/next.js → https://context7.com/vercel/next.js/llms.txt +- https://github.com/shadcn-ui/ui → https://context7.com/shadcn-ui/ui/llms.txt +``` + +Websites: +``` +Pattern: https://context7.com/websites/{normalized-path}/llms.txt +Examples: +- https://docs.imgix.com/ → https://context7.com/websites/imgix/llms.txt +- https://ffmpeg.org/doxygen/8.0/ → https://context7.com/websites/ffmpeg_doxygen_8_0/llms.txt +``` + +Topic-specific searches: +``` +Pattern: https://context7.com/{path}/llms.txt?topic={query} +Examples: +- https://context7.com/shadcn-ui/ui/llms.txt?topic=date +- https://context7.com/vercel/next.js/llms.txt?topic=cache +- https://context7.com/websites/ffmpeg_doxygen_8_0/llms.txt?topic=compress +``` + +**Benefits:** +- Comprehensive aggregator of documentation +- Up-to-date content +- Topic filtering for targeted results +- Consistent format across libraries +- Reduces search time + +**When to use:** +- ALWAYS try context7.com first before WebSearch +- Use topic parameter when user asks about specific feature +- Fall back to WebSearch only if context7.com returns 404 + +## WebSearch + +**Use when:** +- context7.com unavailable or returns 404 +- Finding GitHub repository URLs +- Locating official documentation sites +- Identifying package registries +- Searching for specific versions + +**Best practices:** +- Try context7.com FIRST +- Include domain in query: `site:docs.example.com` +- Specify version when needed: `v2.0 llms.txt` +- Use official terms: "official repository" "documentation" +- Check multiple domains if first fails + +**Example queries:** +``` +Good: "Next.js llms.txt site:nextjs.org" +Good: "React v18 documentation site:react.dev" +Good: "Vue 3 official github repository" + +Avoid: "how to use react" (too vague) +Avoid: "best react tutorial" (not official) +``` + +## WebFetch + +**Use when:** +- Reading llms.txt content +- Accessing single documentation pages +- Retrieving specific URLs +- Checking documentation structure +- Verifying content availability + +**Best practices:** +- Use specific prompt: "Extract all documentation URLs" +- Handle redirects properly +- Check for rate limiting +- Verify content is complete +- Note last-modified dates when available + +**Limitations:** +- Single URL at a time (use Explorer for multiple) +- May timeout on very large pages +- Cannot handle dynamic content +- No JavaScript execution + +## Task Tool with Explore Subagent + +**Use when:** +- Multiple URLs to read (3+) +- Need parallel exploration +- Comprehensive documentation coverage +- Time-sensitive requests +- Large documentation sets + +**Best practices:** +- Launch all agents in single message +- Distribute workload evenly +- Group related URLs per agent +- Maximum 7 agents per batch +- Provide clear extraction goals + +**Example prompt:** +``` +"Read the following URLs and extract: +1. Installation instructions +2. Core API methods +3. Configuration options +4. Common usage examples + +URLs: +- [url1] +- [url2] +- [url3]" +``` + +## Task Tool with Researcher Subagent + +**Use when:** +- No structured documentation found +- Need diverse information sources +- Community knowledge required +- Scattered documentation +- Comparative analysis needed + +**Best practices:** +- Assign specific research areas per agent +- Request source verification +- Ask for date/version information +- Prioritize official sources +- Cross-reference findings + +**Example prompt:** +``` +"Research [library] focusing on: +1. Official installation methods +2. Common usage patterns +3. Known limitations or issues +4. Community best practices + +Prioritize official sources and note version/date for all findings." +``` + +## Repomix + +**Use when:** +- GitHub repository available +- Need complete codebase analysis +- Documentation scattered in repository +- Want to analyze code structure +- API documentation in code comments + +**Installation:** +```bash +# Check if installed +which repomix + +# Install globally if needed +npm install -g repomix + +# Verify installation +repomix --version +``` + +**Usage:** +```bash +# Basic usage +git clone [repo-url] /tmp/docs-analysis +cd /tmp/docs-analysis +repomix --output repomix-output.xml + +# Focus on specific directory +repomix --include "docs/**" --output docs-only.xml + +# Exclude large files +repomix --exclude "*.png,*.jpg,*.pdf" --output repomix-output.xml +``` + +**When Repomix may fail:** +- Repository > 1GB (too large) +- Requires authentication (private repo) +- Slow network connection +- Limited disk space +- Binary-heavy repository + +**Alternatives if Repomix fails:** +```bash +# Option 1: Focus on docs directory only +repomix --include "docs/**,README.md" --output docs.xml + +# Option 2: Use Explorer agents to read specific files +# Launch agents to read key documentation files directly + +# Option 3: Manual repository exploration +# Read README, then explore /docs directory structure +``` + +## Tool Selection Decision Tree + +``` +Need documentation? + ↓ +Try context7.com first + ↓ +Know GitHub org/repo? + YES → https://context7.com/{org}/{repo}/llms.txt + NO → Continue + ↓ +Know website URL? + YES → https://context7.com/websites/{normalized-path}/llms.txt + NO → Continue + ↓ +Specific topic/feature? + YES → Add ?topic={query} parameter + NO → Use base llms.txt URL + ↓ +context7.com found? + YES → Process llms.txt URLs (go to URL count check) + NO → Continue + ↓ +Fallback: WebSearch for llms.txt + ↓ +Single URL? + YES → WebFetch + NO → Continue + ↓ +1-3 URLs? + YES → Single Explorer agent + NO → Continue + ↓ +4+ URLs? + YES → Multiple Explorer agents (3-7) + NO → Continue + ↓ +Need repository analysis? + YES → Repomix (if available) + NO → Continue + ↓ +No structured docs? + YES → Researcher agents +``` + +## Quick Reference + +| Tool | Best For | Speed | Coverage | Complexity | +|------|----------|-------|----------|------------| +| context7.com | llms.txt lookup | Instant | High | Low | +| context7.com?topic= | Targeted search | Instant | Focused | Low | +| WebSearch | Finding URLs | Fast | Narrow | Low | +| WebFetch | Single page | Fast | Single | Low | +| Explorer | Multiple URLs | Fast | Medium | Medium | +| Researcher | Scattered info | Slow | Wide | High | +| Repomix | Repository | Medium | Complete | Medium | diff --git a/skills/document-skills/docx/LICENSE.txt b/skills/document-skills/docx/LICENSE.txt new file mode 100644 index 0000000..c55ab42 --- /dev/null +++ b/skills/document-skills/docx/LICENSE.txt @@ -0,0 +1,30 @@ +© 2025 Anthropic, PBC. All rights reserved. + +LICENSE: Use of these materials (including all code, prompts, assets, files, +and other components of this Skill) is governed by your agreement with +Anthropic regarding use of Anthropic's services. If no separate agreement +exists, use is governed by Anthropic's Consumer Terms of Service or +Commercial Terms of Service, as applicable: +https://www.anthropic.com/legal/consumer-terms +https://www.anthropic.com/legal/commercial-terms +Your applicable agreement is referred to as the "Agreement." "Services" are +as defined in the Agreement. + +ADDITIONAL RESTRICTIONS: Notwithstanding anything in the Agreement to the +contrary, users may not: + +- Extract these materials from the Services or retain copies of these + materials outside the Services +- Reproduce or copy these materials, except for temporary copies created + automatically during authorized use of the Services +- Create derivative works based on these materials +- Distribute, sublicense, or transfer these materials to any third party +- Make, offer to sell, sell, or import any inventions embodied in these + materials +- Reverse engineer, decompile, or disassemble these materials + +The receipt, viewing, or possession of these materials does not convey or +imply any license or right beyond those expressly granted above. + +Anthropic retains all right, title, and interest in these materials, +including all copyrights, patents, and other intellectual property rights. diff --git a/skills/document-skills/docx/SKILL.md b/skills/document-skills/docx/SKILL.md new file mode 100644 index 0000000..6646638 --- /dev/null +++ b/skills/document-skills/docx/SKILL.md @@ -0,0 +1,197 @@ +--- +name: docx +description: "Comprehensive document creation, editing, and analysis with support for tracked changes, comments, formatting preservation, and text extraction. When Claude needs to work with professional documents (.docx files) for: (1) Creating new documents, (2) Modifying or editing content, (3) Working with tracked changes, (4) Adding comments, or any other document tasks" +license: Proprietary. LICENSE.txt has complete terms +--- + +# DOCX creation, editing, and analysis + +## Overview + +A user may ask you to create, edit, or analyze the contents of a .docx file. A .docx file is essentially a ZIP archive containing XML files and other resources that you can read or edit. You have different tools and workflows available for different tasks. + +## Workflow Decision Tree + +### Reading/Analyzing Content +Use "Text extraction" or "Raw XML access" sections below + +### Creating New Document +Use "Creating a new Word document" workflow + +### Editing Existing Document +- **Your own document + simple changes** + Use "Basic OOXML editing" workflow + +- **Someone else's document** + Use **"Redlining workflow"** (recommended default) + +- **Legal, academic, business, or government docs** + Use **"Redlining workflow"** (required) + +## Reading and analyzing content + +### Text extraction +If you just need to read the text contents of a document, you should convert the document to markdown using pandoc. Pandoc provides excellent support for preserving document structure and can show tracked changes: + +```bash +# Convert document to markdown with tracked changes +pandoc --track-changes=all path-to-file.docx -o output.md +# Options: --track-changes=accept/reject/all +``` + +### Raw XML access +You need raw XML access for: comments, complex formatting, document structure, embedded media, and metadata. For any of these features, you'll need to unpack a document and read its raw XML contents. + +#### Unpacking a file +`python ooxml/scripts/unpack.py ` + +#### Key file structures +* `word/document.xml` - Main document contents +* `word/comments.xml` - Comments referenced in document.xml +* `word/media/` - Embedded images and media files +* Tracked changes use `` (insertions) and `` (deletions) tags + +## Creating a new Word document + +When creating a new Word document from scratch, use **docx-js**, which allows you to create Word documents using JavaScript/TypeScript. + +### Workflow +1. **MANDATORY - READ ENTIRE FILE**: Read [`docx-js.md`](docx-js.md) (~500 lines) completely from start to finish. **NEVER set any range limits when reading this file.** Read the full file content for detailed syntax, critical formatting rules, and best practices before proceeding with document creation. +2. Create a JavaScript/TypeScript file using Document, Paragraph, TextRun components (You can assume all dependencies are installed, but if not, refer to the dependencies section below) +3. Export as .docx using Packer.toBuffer() + +## Editing an existing Word document + +When editing an existing Word document, use the **Document library** (a Python library for OOXML manipulation). The library automatically handles infrastructure setup and provides methods for document manipulation. For complex scenarios, you can access the underlying DOM directly through the library. + +### Workflow +1. **MANDATORY - READ ENTIRE FILE**: Read [`ooxml.md`](ooxml.md) (~600 lines) completely from start to finish. **NEVER set any range limits when reading this file.** Read the full file content for the Document library API and XML patterns for directly editing document files. +2. Unpack the document: `python ooxml/scripts/unpack.py ` +3. Create and run a Python script using the Document library (see "Document Library" section in ooxml.md) +4. Pack the final document: `python ooxml/scripts/pack.py ` + +The Document library provides both high-level methods for common operations and direct DOM access for complex scenarios. + +## Redlining workflow for document review + +This workflow allows you to plan comprehensive tracked changes using markdown before implementing them in OOXML. **CRITICAL**: For complete tracked changes, you must implement ALL changes systematically. + +**Batching Strategy**: Group related changes into batches of 3-10 changes. This makes debugging manageable while maintaining efficiency. Test each batch before moving to the next. + +**Principle: Minimal, Precise Edits** +When implementing tracked changes, only mark text that actually changes. Repeating unchanged text makes edits harder to review and appears unprofessional. Break replacements into: [unchanged text] + [deletion] + [insertion] + [unchanged text]. Preserve the original run's RSID for unchanged text by extracting the `` element from the original and reusing it. + +Example - Changing "30 days" to "60 days" in a sentence: +```python +# BAD - Replaces entire sentence +'The term is 30 days.The term is 60 days.' + +# GOOD - Only marks what changed, preserves original for unchanged text +'The term is 3060 days.' +``` + +### Tracked changes workflow + +1. **Get markdown representation**: Convert document to markdown with tracked changes preserved: + ```bash + pandoc --track-changes=all path-to-file.docx -o current.md + ``` + +2. **Identify and group changes**: Review the document and identify ALL changes needed, organizing them into logical batches: + + **Location methods** (for finding changes in XML): + - Section/heading numbers (e.g., "Section 3.2", "Article IV") + - Paragraph identifiers if numbered + - Grep patterns with unique surrounding text + - Document structure (e.g., "first paragraph", "signature block") + - **DO NOT use markdown line numbers** - they don't map to XML structure + + **Batch organization** (group 3-10 related changes per batch): + - By section: "Batch 1: Section 2 amendments", "Batch 2: Section 5 updates" + - By type: "Batch 1: Date corrections", "Batch 2: Party name changes" + - By complexity: Start with simple text replacements, then tackle complex structural changes + - Sequential: "Batch 1: Pages 1-3", "Batch 2: Pages 4-6" + +3. **Read documentation and unpack**: + - **MANDATORY - READ ENTIRE FILE**: Read [`ooxml.md`](ooxml.md) (~600 lines) completely from start to finish. **NEVER set any range limits when reading this file.** Pay special attention to the "Document Library" and "Tracked Change Patterns" sections. + - **Unpack the document**: `python ooxml/scripts/unpack.py ` + - **Note the suggested RSID**: The unpack script will suggest an RSID to use for your tracked changes. Copy this RSID for use in step 4b. + +4. **Implement changes in batches**: Group changes logically (by section, by type, or by proximity) and implement them together in a single script. This approach: + - Makes debugging easier (smaller batch = easier to isolate errors) + - Allows incremental progress + - Maintains efficiency (batch size of 3-10 changes works well) + + **Suggested batch groupings:** + - By document section (e.g., "Section 3 changes", "Definitions", "Termination clause") + - By change type (e.g., "Date changes", "Party name updates", "Legal term replacements") + - By proximity (e.g., "Changes on pages 1-3", "Changes in first half of document") + + For each batch of related changes: + + **a. Map text to XML**: Grep for text in `word/document.xml` to verify how text is split across `` elements. + + **b. Create and run script**: Use `get_node` to find nodes, implement changes, then `doc.save()`. See **"Document Library"** section in ooxml.md for patterns. + + **Note**: Always grep `word/document.xml` immediately before writing a script to get current line numbers and verify text content. Line numbers change after each script run. + +5. **Pack the document**: After all batches are complete, convert the unpacked directory back to .docx: + ```bash + python ooxml/scripts/pack.py unpacked reviewed-document.docx + ``` + +6. **Final verification**: Do a comprehensive check of the complete document: + - Convert final document to markdown: + ```bash + pandoc --track-changes=all reviewed-document.docx -o verification.md + ``` + - Verify ALL changes were applied correctly: + ```bash + grep "original phrase" verification.md # Should NOT find it + grep "replacement phrase" verification.md # Should find it + ``` + - Check that no unintended changes were introduced + + +## Converting Documents to Images + +To visually analyze Word documents, convert them to images using a two-step process: + +1. **Convert DOCX to PDF**: + ```bash + soffice --headless --convert-to pdf document.docx + ``` + +2. **Convert PDF pages to JPEG images**: + ```bash + pdftoppm -jpeg -r 150 document.pdf page + ``` + This creates files like `page-1.jpg`, `page-2.jpg`, etc. + +Options: +- `-r 150`: Sets resolution to 150 DPI (adjust for quality/size balance) +- `-jpeg`: Output JPEG format (use `-png` for PNG if preferred) +- `-f N`: First page to convert (e.g., `-f 2` starts from page 2) +- `-l N`: Last page to convert (e.g., `-l 5` stops at page 5) +- `page`: Prefix for output files + +Example for specific range: +```bash +pdftoppm -jpeg -r 150 -f 2 -l 5 document.pdf page # Converts only pages 2-5 +``` + +## Code Style Guidelines +**IMPORTANT**: When generating code for DOCX operations: +- Write concise code +- Avoid verbose variable names and redundant operations +- Avoid unnecessary print statements + +## Dependencies + +Required dependencies (install if not available): + +- **pandoc**: `sudo apt-get install pandoc` (for text extraction) +- **docx**: `npm install -g docx` (for creating new documents) +- **LibreOffice**: `sudo apt-get install libreoffice` (for PDF conversion) +- **Poppler**: `sudo apt-get install poppler-utils` (for pdftoppm to convert PDF to images) +- **defusedxml**: `pip install defusedxml` (for secure XML parsing) \ No newline at end of file diff --git a/skills/document-skills/docx/docx-js.md b/skills/document-skills/docx/docx-js.md new file mode 100644 index 0000000..c6d7b2d --- /dev/null +++ b/skills/document-skills/docx/docx-js.md @@ -0,0 +1,350 @@ +# DOCX Library Tutorial + +Generate .docx files with JavaScript/TypeScript. + +**Important: Read this entire document before starting.** Critical formatting rules and common pitfalls are covered throughout - skipping sections may result in corrupted files or rendering issues. + +## Setup +Assumes docx is already installed globally +If not installed: `npm install -g docx` + +```javascript +const { Document, Packer, Paragraph, TextRun, Table, TableRow, TableCell, ImageRun, Media, + Header, Footer, AlignmentType, PageOrientation, LevelFormat, ExternalHyperlink, + InternalHyperlink, TableOfContents, HeadingLevel, BorderStyle, WidthType, TabStopType, + TabStopPosition, UnderlineType, ShadingType, VerticalAlign, SymbolRun, PageNumber, + FootnoteReferenceRun, Footnote, PageBreak } = require('docx'); + +// Create & Save +const doc = new Document({ sections: [{ children: [/* content */] }] }); +Packer.toBuffer(doc).then(buffer => fs.writeFileSync("doc.docx", buffer)); // Node.js +Packer.toBlob(doc).then(blob => { /* download logic */ }); // Browser +``` + +## Text & Formatting +```javascript +// IMPORTANT: Never use \n for line breaks - always use separate Paragraph elements +// ❌ WRONG: new TextRun("Line 1\nLine 2") +// ✅ CORRECT: new Paragraph({ children: [new TextRun("Line 1")] }), new Paragraph({ children: [new TextRun("Line 2")] }) + +// Basic text with all formatting options +new Paragraph({ + alignment: AlignmentType.CENTER, + spacing: { before: 200, after: 200 }, + indent: { left: 720, right: 720 }, + children: [ + new TextRun({ text: "Bold", bold: true }), + new TextRun({ text: "Italic", italics: true }), + new TextRun({ text: "Underlined", underline: { type: UnderlineType.DOUBLE, color: "FF0000" } }), + new TextRun({ text: "Colored", color: "FF0000", size: 28, font: "Arial" }), // Arial default + new TextRun({ text: "Highlighted", highlight: "yellow" }), + new TextRun({ text: "Strikethrough", strike: true }), + new TextRun({ text: "x2", superScript: true }), + new TextRun({ text: "H2O", subScript: true }), + new TextRun({ text: "SMALL CAPS", smallCaps: true }), + new SymbolRun({ char: "2022", font: "Symbol" }), // Bullet • + new SymbolRun({ char: "00A9", font: "Arial" }) // Copyright © - Arial for symbols + ] +}) +``` + +## Styles & Professional Formatting + +```javascript +const doc = new Document({ + styles: { + default: { document: { run: { font: "Arial", size: 24 } } }, // 12pt default + paragraphStyles: [ + // Document title style - override built-in Title style + { id: "Title", name: "Title", basedOn: "Normal", + run: { size: 56, bold: true, color: "000000", font: "Arial" }, + paragraph: { spacing: { before: 240, after: 120 }, alignment: AlignmentType.CENTER } }, + // IMPORTANT: Override built-in heading styles by using their exact IDs + { id: "Heading1", name: "Heading 1", basedOn: "Normal", next: "Normal", quickFormat: true, + run: { size: 32, bold: true, color: "000000", font: "Arial" }, // 16pt + paragraph: { spacing: { before: 240, after: 240 }, outlineLevel: 0 } }, // Required for TOC + { id: "Heading2", name: "Heading 2", basedOn: "Normal", next: "Normal", quickFormat: true, + run: { size: 28, bold: true, color: "000000", font: "Arial" }, // 14pt + paragraph: { spacing: { before: 180, after: 180 }, outlineLevel: 1 } }, + // Custom styles use your own IDs + { id: "myStyle", name: "My Style", basedOn: "Normal", + run: { size: 28, bold: true, color: "000000" }, + paragraph: { spacing: { after: 120 }, alignment: AlignmentType.CENTER } } + ], + characterStyles: [{ id: "myCharStyle", name: "My Char Style", + run: { color: "FF0000", bold: true, underline: { type: UnderlineType.SINGLE } } }] + }, + sections: [{ + properties: { page: { margin: { top: 1440, right: 1440, bottom: 1440, left: 1440 } } }, + children: [ + new Paragraph({ heading: HeadingLevel.TITLE, children: [new TextRun("Document Title")] }), // Uses overridden Title style + new Paragraph({ heading: HeadingLevel.HEADING_1, children: [new TextRun("Heading 1")] }), // Uses overridden Heading1 style + new Paragraph({ style: "myStyle", children: [new TextRun("Custom paragraph style")] }), + new Paragraph({ children: [ + new TextRun("Normal with "), + new TextRun({ text: "custom char style", style: "myCharStyle" }) + ]}) + ] + }] +}); +``` + +**Professional Font Combinations:** +- **Arial (Headers) + Arial (Body)** - Most universally supported, clean and professional +- **Times New Roman (Headers) + Arial (Body)** - Classic serif headers with modern sans-serif body +- **Georgia (Headers) + Verdana (Body)** - Optimized for screen reading, elegant contrast + +**Key Styling Principles:** +- **Override built-in styles**: Use exact IDs like "Heading1", "Heading2", "Heading3" to override Word's built-in heading styles +- **HeadingLevel constants**: `HeadingLevel.HEADING_1` uses "Heading1" style, `HeadingLevel.HEADING_2` uses "Heading2" style, etc. +- **Include outlineLevel**: Set `outlineLevel: 0` for H1, `outlineLevel: 1` for H2, etc. to ensure TOC works correctly +- **Use custom styles** instead of inline formatting for consistency +- **Set a default font** using `styles.default.document.run.font` - Arial is universally supported +- **Establish visual hierarchy** with different font sizes (titles > headers > body) +- **Add proper spacing** with `before` and `after` paragraph spacing +- **Use colors sparingly**: Default to black (000000) and shades of gray for titles and headings (heading 1, heading 2, etc.) +- **Set consistent margins** (1440 = 1 inch is standard) + + +## Lists (ALWAYS USE PROPER LISTS - NEVER USE UNICODE BULLETS) +```javascript +// Bullets - ALWAYS use the numbering config, NOT unicode symbols +// CRITICAL: Use LevelFormat.BULLET constant, NOT the string "bullet" +const doc = new Document({ + numbering: { + config: [ + { reference: "bullet-list", + levels: [{ level: 0, format: LevelFormat.BULLET, text: "•", alignment: AlignmentType.LEFT, + style: { paragraph: { indent: { left: 720, hanging: 360 } } } }] }, + { reference: "first-numbered-list", + levels: [{ level: 0, format: LevelFormat.DECIMAL, text: "%1.", alignment: AlignmentType.LEFT, + style: { paragraph: { indent: { left: 720, hanging: 360 } } } }] }, + { reference: "second-numbered-list", // Different reference = restarts at 1 + levels: [{ level: 0, format: LevelFormat.DECIMAL, text: "%1.", alignment: AlignmentType.LEFT, + style: { paragraph: { indent: { left: 720, hanging: 360 } } } }] } + ] + }, + sections: [{ + children: [ + // Bullet list items + new Paragraph({ numbering: { reference: "bullet-list", level: 0 }, + children: [new TextRun("First bullet point")] }), + new Paragraph({ numbering: { reference: "bullet-list", level: 0 }, + children: [new TextRun("Second bullet point")] }), + // Numbered list items + new Paragraph({ numbering: { reference: "first-numbered-list", level: 0 }, + children: [new TextRun("First numbered item")] }), + new Paragraph({ numbering: { reference: "first-numbered-list", level: 0 }, + children: [new TextRun("Second numbered item")] }), + // ⚠️ CRITICAL: Different reference = INDEPENDENT list that restarts at 1 + // Same reference = CONTINUES previous numbering + new Paragraph({ numbering: { reference: "second-numbered-list", level: 0 }, + children: [new TextRun("Starts at 1 again (because different reference)")] }) + ] + }] +}); + +// ⚠️ CRITICAL NUMBERING RULE: Each reference creates an INDEPENDENT numbered list +// - Same reference = continues numbering (1, 2, 3... then 4, 5, 6...) +// - Different reference = restarts at 1 (1, 2, 3... then 1, 2, 3...) +// Use unique reference names for each separate numbered section! + +// ⚠️ CRITICAL: NEVER use unicode bullets - they create fake lists that don't work properly +// new TextRun("• Item") // WRONG +// new SymbolRun({ char: "2022" }) // WRONG +// ✅ ALWAYS use numbering config with LevelFormat.BULLET for real Word lists +``` + +## Tables +```javascript +// Complete table with margins, borders, headers, and bullet points +const tableBorder = { style: BorderStyle.SINGLE, size: 1, color: "CCCCCC" }; +const cellBorders = { top: tableBorder, bottom: tableBorder, left: tableBorder, right: tableBorder }; + +new Table({ + columnWidths: [4680, 4680], // ⚠️ CRITICAL: Set column widths at table level - values in DXA (twentieths of a point) + margins: { top: 100, bottom: 100, left: 180, right: 180 }, // Set once for all cells + rows: [ + new TableRow({ + tableHeader: true, + children: [ + new TableCell({ + borders: cellBorders, + width: { size: 4680, type: WidthType.DXA }, // ALSO set width on each cell + // ⚠️ CRITICAL: Always use ShadingType.CLEAR to prevent black backgrounds in Word. + shading: { fill: "D5E8F0", type: ShadingType.CLEAR }, + verticalAlign: VerticalAlign.CENTER, + children: [new Paragraph({ + alignment: AlignmentType.CENTER, + children: [new TextRun({ text: "Header", bold: true, size: 22 })] + })] + }), + new TableCell({ + borders: cellBorders, + width: { size: 4680, type: WidthType.DXA }, // ALSO set width on each cell + shading: { fill: "D5E8F0", type: ShadingType.CLEAR }, + children: [new Paragraph({ + alignment: AlignmentType.CENTER, + children: [new TextRun({ text: "Bullet Points", bold: true, size: 22 })] + })] + }) + ] + }), + new TableRow({ + children: [ + new TableCell({ + borders: cellBorders, + width: { size: 4680, type: WidthType.DXA }, // ALSO set width on each cell + children: [new Paragraph({ children: [new TextRun("Regular data")] })] + }), + new TableCell({ + borders: cellBorders, + width: { size: 4680, type: WidthType.DXA }, // ALSO set width on each cell + children: [ + new Paragraph({ + numbering: { reference: "bullet-list", level: 0 }, + children: [new TextRun("First bullet point")] + }), + new Paragraph({ + numbering: { reference: "bullet-list", level: 0 }, + children: [new TextRun("Second bullet point")] + }) + ] + }) + ] + }) + ] +}) +``` + +**IMPORTANT: Table Width & Borders** +- Use BOTH `columnWidths: [width1, width2, ...]` array AND `width: { size: X, type: WidthType.DXA }` on each cell +- Values in DXA (twentieths of a point): 1440 = 1 inch, Letter usable width = 9360 DXA (with 1" margins) +- Apply borders to individual `TableCell` elements, NOT the `Table` itself + +**Precomputed Column Widths (Letter size with 1" margins = 9360 DXA total):** +- **2 columns:** `columnWidths: [4680, 4680]` (equal width) +- **3 columns:** `columnWidths: [3120, 3120, 3120]` (equal width) + +## Links & Navigation +```javascript +// TOC (requires headings) - CRITICAL: Use HeadingLevel only, NOT custom styles +// ❌ WRONG: new Paragraph({ heading: HeadingLevel.HEADING_1, style: "customHeader", children: [new TextRun("Title")] }) +// ✅ CORRECT: new Paragraph({ heading: HeadingLevel.HEADING_1, children: [new TextRun("Title")] }) +new TableOfContents("Table of Contents", { hyperlink: true, headingStyleRange: "1-3" }), + +// External link +new Paragraph({ + children: [new ExternalHyperlink({ + children: [new TextRun({ text: "Google", style: "Hyperlink" })], + link: "https://www.google.com" + })] +}), + +// Internal link & bookmark +new Paragraph({ + children: [new InternalHyperlink({ + children: [new TextRun({ text: "Go to Section", style: "Hyperlink" })], + anchor: "section1" + })] +}), +new Paragraph({ + children: [new TextRun("Section Content")], + bookmark: { id: "section1", name: "section1" } +}), +``` + +## Images & Media +```javascript +// Basic image with sizing & positioning +// CRITICAL: Always specify 'type' parameter - it's REQUIRED for ImageRun +new Paragraph({ + alignment: AlignmentType.CENTER, + children: [new ImageRun({ + type: "png", // NEW REQUIREMENT: Must specify image type (png, jpg, jpeg, gif, bmp, svg) + data: fs.readFileSync("image.png"), + transformation: { width: 200, height: 150, rotation: 0 }, // rotation in degrees + altText: { title: "Logo", description: "Company logo", name: "Name" } // IMPORTANT: All three fields are required + })] +}) +``` + +## Page Breaks +```javascript +// Manual page break +new Paragraph({ children: [new PageBreak()] }), + +// Page break before paragraph +new Paragraph({ + pageBreakBefore: true, + children: [new TextRun("This starts on a new page")] +}) + +// ⚠️ CRITICAL: NEVER use PageBreak standalone - it will create invalid XML that Word cannot open +// ❌ WRONG: new PageBreak() +// ✅ CORRECT: new Paragraph({ children: [new PageBreak()] }) +``` + +## Headers/Footers & Page Setup +```javascript +const doc = new Document({ + sections: [{ + properties: { + page: { + margin: { top: 1440, right: 1440, bottom: 1440, left: 1440 }, // 1440 = 1 inch + size: { orientation: PageOrientation.LANDSCAPE }, + pageNumbers: { start: 1, formatType: "decimal" } // "upperRoman", "lowerRoman", "upperLetter", "lowerLetter" + } + }, + headers: { + default: new Header({ children: [new Paragraph({ + alignment: AlignmentType.RIGHT, + children: [new TextRun("Header Text")] + })] }) + }, + footers: { + default: new Footer({ children: [new Paragraph({ + alignment: AlignmentType.CENTER, + children: [new TextRun("Page "), new TextRun({ children: [PageNumber.CURRENT] }), new TextRun(" of "), new TextRun({ children: [PageNumber.TOTAL_PAGES] })] + })] }) + }, + children: [/* content */] + }] +}); +``` + +## Tabs +```javascript +new Paragraph({ + tabStops: [ + { type: TabStopType.LEFT, position: TabStopPosition.MAX / 4 }, + { type: TabStopType.CENTER, position: TabStopPosition.MAX / 2 }, + { type: TabStopType.RIGHT, position: TabStopPosition.MAX * 3 / 4 } + ], + children: [new TextRun("Left\tCenter\tRight")] +}) +``` + +## Constants & Quick Reference +- **Underlines:** `SINGLE`, `DOUBLE`, `WAVY`, `DASH` +- **Borders:** `SINGLE`, `DOUBLE`, `DASHED`, `DOTTED` +- **Numbering:** `DECIMAL` (1,2,3), `UPPER_ROMAN` (I,II,III), `LOWER_LETTER` (a,b,c) +- **Tabs:** `LEFT`, `CENTER`, `RIGHT`, `DECIMAL` +- **Symbols:** `"2022"` (•), `"00A9"` (©), `"00AE"` (®), `"2122"` (™), `"00B0"` (°), `"F070"` (✓), `"F0FC"` (✗) + +## Critical Issues & Common Mistakes +- **CRITICAL: PageBreak must ALWAYS be inside a Paragraph** - standalone PageBreak creates invalid XML that Word cannot open +- **ALWAYS use ShadingType.CLEAR for table cell shading** - Never use ShadingType.SOLID (causes black background). +- Measurements in DXA (1440 = 1 inch) | Each table cell needs ≥1 Paragraph | TOC requires HeadingLevel styles only +- **ALWAYS use custom styles** with Arial font for professional appearance and proper visual hierarchy +- **ALWAYS set a default font** using `styles.default.document.run.font` - Arial recommended +- **ALWAYS use columnWidths array for tables** + individual cell widths for compatibility +- **NEVER use unicode symbols for bullets** - always use proper numbering configuration with `LevelFormat.BULLET` constant (NOT the string "bullet") +- **NEVER use \n for line breaks anywhere** - always use separate Paragraph elements for each line +- **ALWAYS use TextRun objects within Paragraph children** - never use text property directly on Paragraph +- **CRITICAL for images**: ImageRun REQUIRES `type` parameter - always specify "png", "jpg", "jpeg", "gif", "bmp", or "svg" +- **CRITICAL for bullets**: Must use `LevelFormat.BULLET` constant, not string "bullet", and include `text: "•"` for the bullet character +- **CRITICAL for numbering**: Each numbering reference creates an INDEPENDENT list. Same reference = continues numbering (1,2,3 then 4,5,6). Different reference = restarts at 1 (1,2,3 then 1,2,3). Use unique reference names for each separate numbered section! +- **CRITICAL for TOC**: When using TableOfContents, headings must use HeadingLevel ONLY - do NOT add custom styles to heading paragraphs or TOC will break +- **Tables**: Set `columnWidths` array + individual cell widths, apply borders to cells not table +- **Set table margins at TABLE level** for consistent cell padding (avoids repetition per cell) \ No newline at end of file diff --git a/skills/document-skills/docx/ooxml.md b/skills/document-skills/docx/ooxml.md new file mode 100644 index 0000000..7677e7b --- /dev/null +++ b/skills/document-skills/docx/ooxml.md @@ -0,0 +1,610 @@ +# Office Open XML Technical Reference + +**Important: Read this entire document before starting.** This document covers: +- [Technical Guidelines](#technical-guidelines) - Schema compliance rules and validation requirements +- [Document Content Patterns](#document-content-patterns) - XML patterns for headings, lists, tables, formatting, etc. +- [Document Library (Python)](#document-library-python) - Recommended approach for OOXML manipulation with automatic infrastructure setup +- [Tracked Changes (Redlining)](#tracked-changes-redlining) - XML patterns for implementing tracked changes + +## Technical Guidelines + +### Schema Compliance +- **Element ordering in ``**: ``, ``, ``, ``, `` +- **Whitespace**: Add `xml:space='preserve'` to `` elements with leading/trailing spaces +- **Unicode**: Escape characters in ASCII content: `"` becomes `“` + - **Character encoding reference**: Curly quotes `""` become `“”`, apostrophe `'` becomes `’`, em-dash `—` becomes `—` +- **Tracked changes**: Use `` and `` tags with `w:author="Claude"` outside `` elements + - **Critical**: `` closes with ``, `` closes with `` - never mix + - **RSIDs must be 8-digit hex**: Use values like `00AB1234` (only 0-9, A-F characters) + - **trackRevisions placement**: Add `` after `` in settings.xml +- **Images**: Add to `word/media/`, reference in `document.xml`, set dimensions to prevent overflow + +## Document Content Patterns + +### Basic Structure +```xml + + Text content + +``` + +### Headings and Styles +```xml + + + + + + Document Title + + + + + Section Heading + +``` + +### Text Formatting +```xml + +Bold + +Italic + +Underlined + +Highlighted +``` + +### Lists +```xml + + + + + + + + First item + + + + + + + + + + New list item 1 + + + + + + + + + + + Bullet item + +``` + +### Tables +```xml + + + + + + + + + + + + Cell 1 + + + + Cell 2 + + + +``` + +### Layout +```xml + + + + + + + + + + + + New Section Title + + + + + + + + + + Centered text + + + + + + + + Monospace text + + + + + + + This text is Courier New + + and this text uses default font + +``` + +## File Updates + +When adding content, update these files: + +**`word/_rels/document.xml.rels`:** +```xml + + +``` + +**`[Content_Types].xml`:** +```xml + + +``` + +### Images +**CRITICAL**: Calculate dimensions to prevent page overflow and maintain aspect ratio. + +```xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +``` + +### Links (Hyperlinks) + +**IMPORTANT**: All hyperlinks (both internal and external) require the Hyperlink style to be defined in styles.xml. Without this style, links will look like regular text instead of blue underlined clickable links. + +**External Links:** +```xml + + + + + Link Text + + + + + +``` + +**Internal Links:** + +```xml + + + + + Link Text + + + + + +Target content + +``` + +**Hyperlink Style (required in styles.xml):** +```xml + + + + + + + + + + +``` + +## Document Library (Python) + +Use the Document class from `scripts/document.py` for all tracked changes and comments. It automatically handles infrastructure setup (people.xml, RSIDs, settings.xml, comment files, relationships, content types). Only use direct XML manipulation for complex scenarios not supported by the library. + +**Working with Unicode and Entities:** +- **Searching**: Both entity notation and Unicode characters work - `contains="“Company"` and `contains="\u201cCompany"` find the same text +- **Replacing**: Use either entities (`“`) or Unicode (`\u201c`) - both work and will be converted appropriately based on the file's encoding (ascii → entities, utf-8 → Unicode) + +### Initialization + +**Find the docx skill root** (directory containing `scripts/` and `ooxml/`): +```bash +# Search for document.py to locate the skill root +# Note: /mnt/skills is used here as an example; check your context for the actual location +find /mnt/skills -name "document.py" -path "*/docx/scripts/*" 2>/dev/null | head -1 +# Example output: /mnt/skills/docx/scripts/document.py +# Skill root is: /mnt/skills/docx +``` + +**Run your script with PYTHONPATH** set to the docx skill root: +```bash +PYTHONPATH=/mnt/skills/docx python your_script.py +``` + +**In your script**, import from the skill root: +```python +from scripts.document import Document, DocxXMLEditor + +# Basic initialization (automatically creates temp copy and sets up infrastructure) +doc = Document('unpacked') + +# Customize author and initials +doc = Document('unpacked', author="John Doe", initials="JD") + +# Enable track revisions mode +doc = Document('unpacked', track_revisions=True) + +# Specify custom RSID (auto-generated if not provided) +doc = Document('unpacked', rsid="07DC5ECB") +``` + +### Creating Tracked Changes + +**CRITICAL**: Only mark text that actually changes. Keep ALL unchanged text outside ``/`` tags. Marking unchanged text makes edits unprofessional and harder to review. + +**Attribute Handling**: The Document class auto-injects attributes (w:id, w:date, w:rsidR, w:rsidDel, w16du:dateUtc, xml:space) into new elements. When preserving unchanged text from the original document, copy the original `` element with its existing attributes to maintain document integrity. + +**Method Selection Guide**: +- **Adding your own changes to regular text**: Use `replace_node()` with ``/`` tags, or `suggest_deletion()` for removing entire `` or `` elements +- **Partially modifying another author's tracked change**: Use `replace_node()` to nest your changes inside their ``/`` +- **Completely rejecting another author's insertion**: Use `revert_insertion()` on the `` element (NOT `suggest_deletion()`) +- **Completely rejecting another author's deletion**: Use `revert_deletion()` on the `` element to restore deleted content using tracked changes + +```python +# Minimal edit - change one word: "The report is monthly" → "The report is quarterly" +# Original: The report is monthly +node = doc["word/document.xml"].get_node(tag="w:r", contains="The report is monthly") +rpr = tags[0].toxml() if (tags := node.getElementsByTagName("w:rPr")) else "" +replacement = f'{rpr}The report is {rpr}monthly{rpr}quarterly' +doc["word/document.xml"].replace_node(node, replacement) + +# Minimal edit - change number: "within 30 days" → "within 45 days" +# Original: within 30 days +node = doc["word/document.xml"].get_node(tag="w:r", contains="within 30 days") +rpr = tags[0].toxml() if (tags := node.getElementsByTagName("w:rPr")) else "" +replacement = f'{rpr}within {rpr}30{rpr}45{rpr} days' +doc["word/document.xml"].replace_node(node, replacement) + +# Complete replacement - preserve formatting even when replacing all text +node = doc["word/document.xml"].get_node(tag="w:r", contains="apple") +rpr = tags[0].toxml() if (tags := node.getElementsByTagName("w:rPr")) else "" +replacement = f'{rpr}apple{rpr}banana orange' +doc["word/document.xml"].replace_node(node, replacement) + +# Insert new content (no attributes needed - auto-injected) +node = doc["word/document.xml"].get_node(tag="w:r", contains="existing text") +doc["word/document.xml"].insert_after(node, 'new text') + +# Partially delete another author's insertion +# Original: quarterly financial report +# Goal: Delete only "financial" to make it "quarterly report" +node = doc["word/document.xml"].get_node(tag="w:ins", attrs={"w:id": "5"}) +# IMPORTANT: Preserve w:author="Jane Smith" on the outer to maintain authorship +replacement = ''' + quarterly + financial + report +''' +doc["word/document.xml"].replace_node(node, replacement) + +# Change part of another author's insertion +# Original: in silence, safe and sound +# Goal: Change "safe and sound" to "soft and unbound" +node = doc["word/document.xml"].get_node(tag="w:ins", attrs={"w:id": "8"}) +replacement = f''' + in silence, + + + soft and unbound + + + safe and sound +''' +doc["word/document.xml"].replace_node(node, replacement) + +# Delete entire run (use only when deleting all content; use replace_node for partial deletions) +node = doc["word/document.xml"].get_node(tag="w:r", contains="text to delete") +doc["word/document.xml"].suggest_deletion(node) + +# Delete entire paragraph (in-place, handles both regular and numbered list paragraphs) +para = doc["word/document.xml"].get_node(tag="w:p", contains="paragraph to delete") +doc["word/document.xml"].suggest_deletion(para) + +# Add new numbered list item +target_para = doc["word/document.xml"].get_node(tag="w:p", contains="existing list item") +pPr = tags[0].toxml() if (tags := target_para.getElementsByTagName("w:pPr")) else "" +new_item = f'{pPr}New item' +tracked_para = DocxXMLEditor.suggest_paragraph(new_item) +doc["word/document.xml"].insert_after(target_para, tracked_para) +# Optional: add spacing paragraph before content for better visual separation +# spacing = DocxXMLEditor.suggest_paragraph('') +# doc["word/document.xml"].insert_after(target_para, spacing + tracked_para) +``` + +### Adding Comments + +```python +# Add comment spanning two existing tracked changes +# Note: w:id is auto-generated. Only search by w:id if you know it from XML inspection +start_node = doc["word/document.xml"].get_node(tag="w:del", attrs={"w:id": "1"}) +end_node = doc["word/document.xml"].get_node(tag="w:ins", attrs={"w:id": "2"}) +doc.add_comment(start=start_node, end=end_node, text="Explanation of this change") + +# Add comment on a paragraph +para = doc["word/document.xml"].get_node(tag="w:p", contains="paragraph text") +doc.add_comment(start=para, end=para, text="Comment on this paragraph") + +# Add comment on newly created tracked change +# First create the tracked change +node = doc["word/document.xml"].get_node(tag="w:r", contains="old") +new_nodes = doc["word/document.xml"].replace_node( + node, + 'oldnew' +) +# Then add comment on the newly created elements +# new_nodes[0] is the , new_nodes[1] is the +doc.add_comment(start=new_nodes[0], end=new_nodes[1], text="Changed old to new per requirements") + +# Reply to existing comment +doc.reply_to_comment(parent_comment_id=0, text="I agree with this change") +``` + +### Rejecting Tracked Changes + +**IMPORTANT**: Use `revert_insertion()` to reject insertions and `revert_deletion()` to restore deletions using tracked changes. Use `suggest_deletion()` only for regular unmarked content. + +```python +# Reject insertion (wraps it in deletion) +# Use this when another author inserted text that you want to delete +ins = doc["word/document.xml"].get_node(tag="w:ins", attrs={"w:id": "5"}) +nodes = doc["word/document.xml"].revert_insertion(ins) # Returns [ins] + +# Reject deletion (creates insertion to restore deleted content) +# Use this when another author deleted text that you want to restore +del_elem = doc["word/document.xml"].get_node(tag="w:del", attrs={"w:id": "3"}) +nodes = doc["word/document.xml"].revert_deletion(del_elem) # Returns [del_elem, new_ins] + +# Reject all insertions in a paragraph +para = doc["word/document.xml"].get_node(tag="w:p", contains="paragraph text") +nodes = doc["word/document.xml"].revert_insertion(para) # Returns [para] + +# Reject all deletions in a paragraph +para = doc["word/document.xml"].get_node(tag="w:p", contains="paragraph text") +nodes = doc["word/document.xml"].revert_deletion(para) # Returns [para] +``` + +### Inserting Images + +**CRITICAL**: The Document class works with a temporary copy at `doc.unpacked_path`. Always copy images to this temp directory, not the original unpacked folder. + +```python +from PIL import Image +import shutil, os + +# Initialize document first +doc = Document('unpacked') + +# Copy image and calculate full-width dimensions with aspect ratio +media_dir = os.path.join(doc.unpacked_path, 'word/media') +os.makedirs(media_dir, exist_ok=True) +shutil.copy('image.png', os.path.join(media_dir, 'image1.png')) +img = Image.open(os.path.join(media_dir, 'image1.png')) +width_emus = int(6.5 * 914400) # 6.5" usable width, 914400 EMUs/inch +height_emus = int(width_emus * img.size[1] / img.size[0]) + +# Add relationship and content type +rels_editor = doc['word/_rels/document.xml.rels'] +next_rid = rels_editor.get_next_rid() +rels_editor.append_to(rels_editor.dom.documentElement, + f'') +doc['[Content_Types].xml'].append_to(doc['[Content_Types].xml'].dom.documentElement, + '') + +# Insert image +node = doc["word/document.xml"].get_node(tag="w:p", line_number=100) +doc["word/document.xml"].insert_after(node, f''' + + + + + + + + + + + + + + + + + +''') +``` + +### Getting Nodes + +```python +# By text content +node = doc["word/document.xml"].get_node(tag="w:p", contains="specific text") + +# By line range +para = doc["word/document.xml"].get_node(tag="w:p", line_number=range(100, 150)) + +# By attributes +node = doc["word/document.xml"].get_node(tag="w:del", attrs={"w:id": "1"}) + +# By exact line number (must be line number where tag opens) +para = doc["word/document.xml"].get_node(tag="w:p", line_number=42) + +# Combine filters +node = doc["word/document.xml"].get_node(tag="w:r", line_number=range(40, 60), contains="text") + +# Disambiguate when text appears multiple times - add line_number range +node = doc["word/document.xml"].get_node(tag="w:r", contains="Section", line_number=range(2400, 2500)) +``` + +### Saving + +```python +# Save with automatic validation (copies back to original directory) +doc.save() # Validates by default, raises error if validation fails + +# Save to different location +doc.save('modified-unpacked') + +# Skip validation (debugging only - needing this in production indicates XML issues) +doc.save(validate=False) +``` + +### Direct DOM Manipulation + +For complex scenarios not covered by the library: + +```python +# Access any XML file +editor = doc["word/document.xml"] +editor = doc["word/comments.xml"] + +# Direct DOM access (defusedxml.minidom.Document) +node = doc["word/document.xml"].get_node(tag="w:p", line_number=5) +parent = node.parentNode +parent.removeChild(node) +parent.appendChild(node) # Move to end + +# General document manipulation (without tracked changes) +old_node = doc["word/document.xml"].get_node(tag="w:p", contains="original text") +doc["word/document.xml"].replace_node(old_node, "replacement text") + +# Multiple insertions - use return value to maintain order +node = doc["word/document.xml"].get_node(tag="w:r", line_number=100) +nodes = doc["word/document.xml"].insert_after(node, "A") +nodes = doc["word/document.xml"].insert_after(nodes[-1], "B") +nodes = doc["word/document.xml"].insert_after(nodes[-1], "C") +# Results in: original_node, A, B, C +``` + +## Tracked Changes (Redlining) + +**Use the Document class above for all tracked changes.** The patterns below are for reference when constructing replacement XML strings. + +### Validation Rules +The validator checks that the document text matches the original after reverting Claude's changes. This means: +- **NEVER modify text inside another author's `` or `` tags** +- **ALWAYS use nested deletions** to remove another author's insertions +- **Every edit must be properly tracked** with `` or `` tags + +### Tracked Change Patterns + +**CRITICAL RULES**: +1. Never modify the content inside another author's tracked changes. Always use nested deletions. +2. **XML Structure**: Always place `` and `` at paragraph level containing complete `` elements. Never nest inside `` elements - this creates invalid XML that breaks document processing. + +**Text Insertion:** +```xml + + + inserted text + + +``` + +**Text Deletion:** +```xml + + + deleted text + + +``` + +**Deleting Another Author's Insertion (MUST use nested structure):** +```xml + + + + monthly + + + + weekly + +``` + +**Restoring Another Author's Deletion:** +```xml + + + within 30 days + + + within 30 days + +``` \ No newline at end of file diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd new file mode 100644 index 0000000..6454ef9 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chart.xsd @@ -0,0 +1,1499 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd new file mode 100644 index 0000000..afa4f46 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-chartDrawing.xsd @@ -0,0 +1,146 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd new file mode 100644 index 0000000..64e66b8 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-diagram.xsd @@ -0,0 +1,1085 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd new file mode 100644 index 0000000..687eea8 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-lockedCanvas.xsd @@ -0,0 +1,11 @@ + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd new file mode 100644 index 0000000..6ac81b0 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-main.xsd @@ -0,0 +1,3081 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd new file mode 100644 index 0000000..1dbf051 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-picture.xsd @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd new file mode 100644 index 0000000..f1af17d --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-spreadsheetDrawing.xsd @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd new file mode 100644 index 0000000..0a185ab --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/dml-wordprocessingDrawing.xsd @@ -0,0 +1,287 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd new file mode 100644 index 0000000..14ef488 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/pml.xsd @@ -0,0 +1,1676 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd new file mode 100644 index 0000000..c20f3bf --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-additionalCharacteristics.xsd @@ -0,0 +1,28 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd new file mode 100644 index 0000000..ac60252 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-bibliography.xsd @@ -0,0 +1,144 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd new file mode 100644 index 0000000..424b8ba --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-commonSimpleTypes.xsd @@ -0,0 +1,174 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd new file mode 100644 index 0000000..2bddce2 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlDataProperties.xsd @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd new file mode 100644 index 0000000..8a8c18b --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-customXmlSchemaProperties.xsd @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd new file mode 100644 index 0000000..5c42706 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd @@ -0,0 +1,59 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd new file mode 100644 index 0000000..853c341 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd new file mode 100644 index 0000000..da835ee --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-documentPropertiesVariantTypes.xsd @@ -0,0 +1,195 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd new file mode 100644 index 0000000..87ad265 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-math.xsd @@ -0,0 +1,582 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd new file mode 100644 index 0000000..9e86f1b --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/shared-relationshipReference.xsd @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd new file mode 100644 index 0000000..d0be42e --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/sml.xsd @@ -0,0 +1,4439 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd new file mode 100644 index 0000000..8821dd1 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-main.xsd @@ -0,0 +1,570 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd new file mode 100644 index 0000000..ca2575c --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-officeDrawing.xsd @@ -0,0 +1,509 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd new file mode 100644 index 0000000..dd079e6 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-presentationDrawing.xsd @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd new file mode 100644 index 0000000..3dd6cf6 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-spreadsheetDrawing.xsd @@ -0,0 +1,108 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd new file mode 100644 index 0000000..f1041e3 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/vml-wordprocessingDrawing.xsd @@ -0,0 +1,96 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd new file mode 100644 index 0000000..9c5b7a6 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/wml.xsd @@ -0,0 +1,3646 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd new file mode 100644 index 0000000..0f13678 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ISO-IEC29500-4_2016/xml.xsd @@ -0,0 +1,116 @@ + + + + + + See http://www.w3.org/XML/1998/namespace.html and + http://www.w3.org/TR/REC-xml for information about this namespace. + + This schema document describes the XML namespace, in a form + suitable for import by other schema documents. + + Note that local names in this namespace are intended to be defined + only by the World Wide Web Consortium or its subgroups. The + following names are currently defined in this namespace and should + not be used with conflicting semantics by any Working Group, + specification, or document instance: + + base (as an attribute name): denotes an attribute whose value + provides a URI to be used as the base for interpreting any + relative URIs in the scope of the element on which it + appears; its value is inherited. This name is reserved + by virtue of its definition in the XML Base specification. + + lang (as an attribute name): denotes an attribute whose value + is a language code for the natural language of the content of + any element; its value is inherited. This name is reserved + by virtue of its definition in the XML specification. + + space (as an attribute name): denotes an attribute whose + value is a keyword indicating what whitespace processing + discipline is intended for the content of the element; its + value is inherited. This name is reserved by virtue of its + definition in the XML specification. + + Father (in any context at all): denotes Jon Bosak, the chair of + the original XML Working Group. This name is reserved by + the following decision of the W3C XML Plenary and + XML Coordination groups: + + In appreciation for his vision, leadership and dedication + the W3C XML Plenary on this 10th day of February, 2000 + reserves for Jon Bosak in perpetuity the XML name + xml:Father + + + + + This schema defines attributes and an attribute group + suitable for use by + schemas wishing to allow xml:base, xml:lang or xml:space attributes + on elements they define. + + To enable this, such a schema must import this schema + for the XML namespace, e.g. as follows: + <schema . . .> + . . . + <import namespace="http://www.w3.org/XML/1998/namespace" + schemaLocation="http://www.w3.org/2001/03/xml.xsd"/> + + Subsequently, qualified reference to any of the attributes + or the group defined below will have the desired effect, e.g. + + <type . . .> + . . . + <attributeGroup ref="xml:specialAttrs"/> + + will define a type which will schema-validate an instance + element with any of those attributes + + + + In keeping with the XML Schema WG's standard versioning + policy, this schema document will persist at + http://www.w3.org/2001/03/xml.xsd. + At the date of issue it can also be found at + http://www.w3.org/2001/xml.xsd. + The schema document at that URI may however change in the future, + in order to remain compatible with the latest version of XML Schema + itself. In other words, if the XML Schema namespace changes, the version + of this document at + http://www.w3.org/2001/xml.xsd will change + accordingly; the version at + http://www.w3.org/2001/03/xml.xsd will not change. + + + + + + In due course, we should install the relevant ISO 2- and 3-letter + codes as the enumerated possible values . . . + + + + + + + + + + + + + + + See http://www.w3.org/TR/xmlbase/ for + information about this attribute. + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd new file mode 100644 index 0000000..a6de9d2 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-contentTypes.xsd @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd new file mode 100644 index 0000000..10e978b --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-coreProperties.xsd @@ -0,0 +1,50 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd new file mode 100644 index 0000000..4248bf7 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-digSig.xsd @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd new file mode 100644 index 0000000..5649746 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/ecma/fouth-edition/opc-relationships.xsd @@ -0,0 +1,33 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/mce/mc.xsd b/skills/document-skills/docx/ooxml/schemas/mce/mc.xsd new file mode 100644 index 0000000..ef72545 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/mce/mc.xsd @@ -0,0 +1,75 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2010.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2010.xsd new file mode 100644 index 0000000..f65f777 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2010.xsd @@ -0,0 +1,560 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2012.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2012.xsd new file mode 100644 index 0000000..6b00755 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2012.xsd @@ -0,0 +1,67 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2018.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2018.xsd new file mode 100644 index 0000000..f321d33 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-2018.xsd @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-cex-2018.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-cex-2018.xsd new file mode 100644 index 0000000..364c6a9 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-cex-2018.xsd @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-cid-2016.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-cid-2016.xsd new file mode 100644 index 0000000..fed9d15 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-cid-2016.xsd @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd new file mode 100644 index 0000000..680cf15 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-sdtdatahash-2020.xsd @@ -0,0 +1,4 @@ + + + + diff --git a/skills/document-skills/docx/ooxml/schemas/microsoft/wml-symex-2015.xsd b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-symex-2015.xsd new file mode 100644 index 0000000..89ada90 --- /dev/null +++ b/skills/document-skills/docx/ooxml/schemas/microsoft/wml-symex-2015.xsd @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/skills/document-skills/docx/ooxml/scripts/pack.py b/skills/document-skills/docx/ooxml/scripts/pack.py new file mode 100755 index 0000000..68bc088 --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/pack.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +""" +Tool to pack a directory into a .docx, .pptx, or .xlsx file with XML formatting undone. + +Example usage: + python pack.py [--force] +""" + +import argparse +import shutil +import subprocess +import sys +import tempfile +import defusedxml.minidom +import zipfile +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser(description="Pack a directory into an Office file") + parser.add_argument("input_directory", help="Unpacked Office document directory") + parser.add_argument("output_file", help="Output Office file (.docx/.pptx/.xlsx)") + parser.add_argument("--force", action="store_true", help="Skip validation") + args = parser.parse_args() + + try: + success = pack_document( + args.input_directory, args.output_file, validate=not args.force + ) + + # Show warning if validation was skipped + if args.force: + print("Warning: Skipped validation, file may be corrupt", file=sys.stderr) + # Exit with error if validation failed + elif not success: + print("Contents would produce a corrupt file.", file=sys.stderr) + print("Please validate XML before repacking.", file=sys.stderr) + print("Use --force to skip validation and pack anyway.", file=sys.stderr) + sys.exit(1) + + except ValueError as e: + sys.exit(f"Error: {e}") + + +def pack_document(input_dir, output_file, validate=False): + """Pack a directory into an Office file (.docx/.pptx/.xlsx). + + Args: + input_dir: Path to unpacked Office document directory + output_file: Path to output Office file + validate: If True, validates with soffice (default: False) + + Returns: + bool: True if successful, False if validation failed + """ + input_dir = Path(input_dir) + output_file = Path(output_file) + + if not input_dir.is_dir(): + raise ValueError(f"{input_dir} is not a directory") + if output_file.suffix.lower() not in {".docx", ".pptx", ".xlsx"}: + raise ValueError(f"{output_file} must be a .docx, .pptx, or .xlsx file") + + # Work in temporary directory to avoid modifying original + with tempfile.TemporaryDirectory() as temp_dir: + temp_content_dir = Path(temp_dir) / "content" + shutil.copytree(input_dir, temp_content_dir) + + # Process XML files to remove pretty-printing whitespace + for pattern in ["*.xml", "*.rels"]: + for xml_file in temp_content_dir.rglob(pattern): + condense_xml(xml_file) + + # Create final Office file as zip archive + output_file.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(output_file, "w", zipfile.ZIP_DEFLATED) as zf: + for f in temp_content_dir.rglob("*"): + if f.is_file(): + zf.write(f, f.relative_to(temp_content_dir)) + + # Validate if requested + if validate: + if not validate_document(output_file): + output_file.unlink() # Delete the corrupt file + return False + + return True + + +def validate_document(doc_path): + """Validate document by converting to HTML with soffice.""" + # Determine the correct filter based on file extension + match doc_path.suffix.lower(): + case ".docx": + filter_name = "html:HTML" + case ".pptx": + filter_name = "html:impress_html_Export" + case ".xlsx": + filter_name = "html:HTML (StarCalc)" + + with tempfile.TemporaryDirectory() as temp_dir: + try: + result = subprocess.run( + [ + "soffice", + "--headless", + "--convert-to", + filter_name, + "--outdir", + temp_dir, + str(doc_path), + ], + capture_output=True, + timeout=10, + text=True, + ) + if not (Path(temp_dir) / f"{doc_path.stem}.html").exists(): + error_msg = result.stderr.strip() or "Document validation failed" + print(f"Validation error: {error_msg}", file=sys.stderr) + return False + return True + except FileNotFoundError: + print("Warning: soffice not found. Skipping validation.", file=sys.stderr) + return True + except subprocess.TimeoutExpired: + print("Validation error: Timeout during conversion", file=sys.stderr) + return False + except Exception as e: + print(f"Validation error: {e}", file=sys.stderr) + return False + + +def condense_xml(xml_file): + """Strip unnecessary whitespace and remove comments.""" + with open(xml_file, "r", encoding="utf-8") as f: + dom = defusedxml.minidom.parse(f) + + # Process each element to remove whitespace and comments + for element in dom.getElementsByTagName("*"): + # Skip w:t elements and their processing + if element.tagName.endswith(":t"): + continue + + # Remove whitespace-only text nodes and comment nodes + for child in list(element.childNodes): + if ( + child.nodeType == child.TEXT_NODE + and child.nodeValue + and child.nodeValue.strip() == "" + ) or child.nodeType == child.COMMENT_NODE: + element.removeChild(child) + + # Write back the condensed XML + with open(xml_file, "wb") as f: + f.write(dom.toxml(encoding="UTF-8")) + + +if __name__ == "__main__": + main() diff --git a/skills/document-skills/docx/ooxml/scripts/unpack.py b/skills/document-skills/docx/ooxml/scripts/unpack.py new file mode 100755 index 0000000..4938798 --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/unpack.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +"""Unpack and format XML contents of Office files (.docx, .pptx, .xlsx)""" + +import random +import sys +import defusedxml.minidom +import zipfile +from pathlib import Path + +# Get command line arguments +assert len(sys.argv) == 3, "Usage: python unpack.py " +input_file, output_dir = sys.argv[1], sys.argv[2] + +# Extract and format +output_path = Path(output_dir) +output_path.mkdir(parents=True, exist_ok=True) +zipfile.ZipFile(input_file).extractall(output_path) + +# Pretty print all XML files +xml_files = list(output_path.rglob("*.xml")) + list(output_path.rglob("*.rels")) +for xml_file in xml_files: + content = xml_file.read_text(encoding="utf-8") + dom = defusedxml.minidom.parseString(content) + xml_file.write_bytes(dom.toprettyxml(indent=" ", encoding="ascii")) + +# For .docx files, suggest an RSID for tracked changes +if input_file.endswith(".docx"): + suggested_rsid = "".join(random.choices("0123456789ABCDEF", k=8)) + print(f"Suggested RSID for edit session: {suggested_rsid}") diff --git a/skills/document-skills/docx/ooxml/scripts/validate.py b/skills/document-skills/docx/ooxml/scripts/validate.py new file mode 100755 index 0000000..508c589 --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/validate.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +""" +Command line tool to validate Office document XML files against XSD schemas and tracked changes. + +Usage: + python validate.py --original +""" + +import argparse +import sys +from pathlib import Path + +from validation import DOCXSchemaValidator, PPTXSchemaValidator, RedliningValidator + + +def main(): + parser = argparse.ArgumentParser(description="Validate Office document XML files") + parser.add_argument( + "unpacked_dir", + help="Path to unpacked Office document directory", + ) + parser.add_argument( + "--original", + required=True, + help="Path to original file (.docx/.pptx/.xlsx)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="Enable verbose output", + ) + args = parser.parse_args() + + # Validate paths + unpacked_dir = Path(args.unpacked_dir) + original_file = Path(args.original) + file_extension = original_file.suffix.lower() + assert unpacked_dir.is_dir(), f"Error: {unpacked_dir} is not a directory" + assert original_file.is_file(), f"Error: {original_file} is not a file" + assert file_extension in [".docx", ".pptx", ".xlsx"], ( + f"Error: {original_file} must be a .docx, .pptx, or .xlsx file" + ) + + # Run validations + match file_extension: + case ".docx": + validators = [DOCXSchemaValidator, RedliningValidator] + case ".pptx": + validators = [PPTXSchemaValidator] + case _: + print(f"Error: Validation not supported for file type {file_extension}") + sys.exit(1) + + # Run validators + success = True + for V in validators: + validator = V(unpacked_dir, original_file, verbose=args.verbose) + if not validator.validate(): + success = False + + if success: + print("All validations PASSED!") + + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/skills/document-skills/docx/ooxml/scripts/validation/__init__.py b/skills/document-skills/docx/ooxml/scripts/validation/__init__.py new file mode 100644 index 0000000..db092ec --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/validation/__init__.py @@ -0,0 +1,15 @@ +""" +Validation modules for Word document processing. +""" + +from .base import BaseSchemaValidator +from .docx import DOCXSchemaValidator +from .pptx import PPTXSchemaValidator +from .redlining import RedliningValidator + +__all__ = [ + "BaseSchemaValidator", + "DOCXSchemaValidator", + "PPTXSchemaValidator", + "RedliningValidator", +] diff --git a/skills/document-skills/docx/ooxml/scripts/validation/base.py b/skills/document-skills/docx/ooxml/scripts/validation/base.py new file mode 100644 index 0000000..0681b19 --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/validation/base.py @@ -0,0 +1,951 @@ +""" +Base validator with common validation logic for document files. +""" + +import re +from pathlib import Path + +import lxml.etree + + +class BaseSchemaValidator: + """Base validator with common validation logic for document files.""" + + # Elements whose 'id' attributes must be unique within their file + # Format: element_name -> (attribute_name, scope) + # scope can be 'file' (unique within file) or 'global' (unique across all files) + UNIQUE_ID_REQUIREMENTS = { + # Word elements + "comment": ("id", "file"), # Comment IDs in comments.xml + "commentrangestart": ("id", "file"), # Must match comment IDs + "commentrangeend": ("id", "file"), # Must match comment IDs + "bookmarkstart": ("id", "file"), # Bookmark start IDs + "bookmarkend": ("id", "file"), # Bookmark end IDs + # Note: ins and del (track changes) can share IDs when part of same revision + # PowerPoint elements + "sldid": ("id", "file"), # Slide IDs in presentation.xml + "sldmasterid": ("id", "global"), # Slide master IDs must be globally unique + "sldlayoutid": ("id", "global"), # Slide layout IDs must be globally unique + "cm": ("authorid", "file"), # Comment author IDs + # Excel elements + "sheet": ("sheetid", "file"), # Sheet IDs in workbook.xml + "definedname": ("id", "file"), # Named range IDs + # Drawing/Shape elements (all formats) + "cxnsp": ("id", "file"), # Connection shape IDs + "sp": ("id", "file"), # Shape IDs + "pic": ("id", "file"), # Picture IDs + "grpsp": ("id", "file"), # Group shape IDs + } + + # Mapping of element names to expected relationship types + # Subclasses should override this with format-specific mappings + ELEMENT_RELATIONSHIP_TYPES = {} + + # Unified schema mappings for all Office document types + SCHEMA_MAPPINGS = { + # Document type specific schemas + "word": "ISO-IEC29500-4_2016/wml.xsd", # Word documents + "ppt": "ISO-IEC29500-4_2016/pml.xsd", # PowerPoint presentations + "xl": "ISO-IEC29500-4_2016/sml.xsd", # Excel spreadsheets + # Common file types + "[Content_Types].xml": "ecma/fouth-edition/opc-contentTypes.xsd", + "app.xml": "ISO-IEC29500-4_2016/shared-documentPropertiesExtended.xsd", + "core.xml": "ecma/fouth-edition/opc-coreProperties.xsd", + "custom.xml": "ISO-IEC29500-4_2016/shared-documentPropertiesCustom.xsd", + ".rels": "ecma/fouth-edition/opc-relationships.xsd", + # Word-specific files + "people.xml": "microsoft/wml-2012.xsd", + "commentsIds.xml": "microsoft/wml-cid-2016.xsd", + "commentsExtensible.xml": "microsoft/wml-cex-2018.xsd", + "commentsExtended.xml": "microsoft/wml-2012.xsd", + # Chart files (common across document types) + "chart": "ISO-IEC29500-4_2016/dml-chart.xsd", + # Theme files (common across document types) + "theme": "ISO-IEC29500-4_2016/dml-main.xsd", + # Drawing and media files + "drawing": "ISO-IEC29500-4_2016/dml-main.xsd", + } + + # Unified namespace constants + MC_NAMESPACE = "http://schemas.openxmlformats.org/markup-compatibility/2006" + XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace" + + # Common OOXML namespaces used across validators + PACKAGE_RELATIONSHIPS_NAMESPACE = ( + "http://schemas.openxmlformats.org/package/2006/relationships" + ) + OFFICE_RELATIONSHIPS_NAMESPACE = ( + "http://schemas.openxmlformats.org/officeDocument/2006/relationships" + ) + CONTENT_TYPES_NAMESPACE = ( + "http://schemas.openxmlformats.org/package/2006/content-types" + ) + + # Folders where we should clean ignorable namespaces + MAIN_CONTENT_FOLDERS = {"word", "ppt", "xl"} + + # All allowed OOXML namespaces (superset of all document types) + OOXML_NAMESPACES = { + "http://schemas.openxmlformats.org/officeDocument/2006/math", + "http://schemas.openxmlformats.org/officeDocument/2006/relationships", + "http://schemas.openxmlformats.org/schemaLibrary/2006/main", + "http://schemas.openxmlformats.org/drawingml/2006/main", + "http://schemas.openxmlformats.org/drawingml/2006/chart", + "http://schemas.openxmlformats.org/drawingml/2006/chartDrawing", + "http://schemas.openxmlformats.org/drawingml/2006/diagram", + "http://schemas.openxmlformats.org/drawingml/2006/picture", + "http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing", + "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing", + "http://schemas.openxmlformats.org/wordprocessingml/2006/main", + "http://schemas.openxmlformats.org/presentationml/2006/main", + "http://schemas.openxmlformats.org/spreadsheetml/2006/main", + "http://schemas.openxmlformats.org/officeDocument/2006/sharedTypes", + "http://www.w3.org/XML/1998/namespace", + } + + def __init__(self, unpacked_dir, original_file, verbose=False): + self.unpacked_dir = Path(unpacked_dir).resolve() + self.original_file = Path(original_file) + self.verbose = verbose + + # Set schemas directory + self.schemas_dir = Path(__file__).parent.parent.parent / "schemas" + + # Get all XML and .rels files + patterns = ["*.xml", "*.rels"] + self.xml_files = [ + f for pattern in patterns for f in self.unpacked_dir.rglob(pattern) + ] + + if not self.xml_files: + print(f"Warning: No XML files found in {self.unpacked_dir}") + + def validate(self): + """Run all validation checks and return True if all pass.""" + raise NotImplementedError("Subclasses must implement the validate method") + + def validate_xml(self): + """Validate that all XML files are well-formed.""" + errors = [] + + for xml_file in self.xml_files: + try: + # Try to parse the XML file + lxml.etree.parse(str(xml_file)) + except lxml.etree.XMLSyntaxError as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {e.lineno}: {e.msg}" + ) + except Exception as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Unexpected error: {str(e)}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} XML violations:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - All XML files are well-formed") + return True + + def validate_namespaces(self): + """Validate that namespace prefixes in Ignorable attributes are declared.""" + errors = [] + + for xml_file in self.xml_files: + try: + root = lxml.etree.parse(str(xml_file)).getroot() + declared = set(root.nsmap.keys()) - {None} # Exclude default namespace + + for attr_val in [ + v for k, v in root.attrib.items() if k.endswith("Ignorable") + ]: + undeclared = set(attr_val.split()) - declared + errors.extend( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Namespace '{ns}' in Ignorable but not declared" + for ns in undeclared + ) + except lxml.etree.XMLSyntaxError: + continue + + if errors: + print(f"FAILED - {len(errors)} namespace issues:") + for error in errors: + print(error) + return False + if self.verbose: + print("PASSED - All namespace prefixes properly declared") + return True + + def validate_unique_ids(self): + """Validate that specific IDs are unique according to OOXML requirements.""" + errors = [] + global_ids = {} # Track globally unique IDs across all files + + for xml_file in self.xml_files: + try: + root = lxml.etree.parse(str(xml_file)).getroot() + file_ids = {} # Track IDs that must be unique within this file + + # Remove all mc:AlternateContent elements from the tree + mc_elements = root.xpath( + ".//mc:AlternateContent", namespaces={"mc": self.MC_NAMESPACE} + ) + for elem in mc_elements: + elem.getparent().remove(elem) + + # Now check IDs in the cleaned tree + for elem in root.iter(): + # Get the element name without namespace + tag = ( + elem.tag.split("}")[-1].lower() + if "}" in elem.tag + else elem.tag.lower() + ) + + # Check if this element type has ID uniqueness requirements + if tag in self.UNIQUE_ID_REQUIREMENTS: + attr_name, scope = self.UNIQUE_ID_REQUIREMENTS[tag] + + # Look for the specified attribute + id_value = None + for attr, value in elem.attrib.items(): + attr_local = ( + attr.split("}")[-1].lower() + if "}" in attr + else attr.lower() + ) + if attr_local == attr_name: + id_value = value + break + + if id_value is not None: + if scope == "global": + # Check global uniqueness + if id_value in global_ids: + prev_file, prev_line, prev_tag = global_ids[ + id_value + ] + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {elem.sourceline}: Global ID '{id_value}' in <{tag}> " + f"already used in {prev_file} at line {prev_line} in <{prev_tag}>" + ) + else: + global_ids[id_value] = ( + xml_file.relative_to(self.unpacked_dir), + elem.sourceline, + tag, + ) + elif scope == "file": + # Check file-level uniqueness + key = (tag, attr_name) + if key not in file_ids: + file_ids[key] = {} + + if id_value in file_ids[key]: + prev_line = file_ids[key][id_value] + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {elem.sourceline}: Duplicate {attr_name}='{id_value}' in <{tag}> " + f"(first occurrence at line {prev_line})" + ) + else: + file_ids[key][id_value] = elem.sourceline + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} ID uniqueness violations:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - All required IDs are unique") + return True + + def validate_file_references(self): + """ + Validate that all .rels files properly reference files and that all files are referenced. + """ + errors = [] + + # Find all .rels files + rels_files = list(self.unpacked_dir.rglob("*.rels")) + + if not rels_files: + if self.verbose: + print("PASSED - No .rels files found") + return True + + # Get all files in the unpacked directory (excluding reference files) + all_files = [] + for file_path in self.unpacked_dir.rglob("*"): + if ( + file_path.is_file() + and file_path.name != "[Content_Types].xml" + and not file_path.name.endswith(".rels") + ): # This file is not referenced by .rels + all_files.append(file_path.resolve()) + + # Track all files that are referenced by any .rels file + all_referenced_files = set() + + if self.verbose: + print( + f"Found {len(rels_files)} .rels files and {len(all_files)} target files" + ) + + # Check each .rels file + for rels_file in rels_files: + try: + # Parse relationships file + rels_root = lxml.etree.parse(str(rels_file)).getroot() + + # Get the directory where this .rels file is located + rels_dir = rels_file.parent + + # Find all relationships and their targets + referenced_files = set() + broken_refs = [] + + for rel in rels_root.findall( + ".//ns:Relationship", + namespaces={"ns": self.PACKAGE_RELATIONSHIPS_NAMESPACE}, + ): + target = rel.get("Target") + if target and not target.startswith( + ("http", "mailto:") + ): # Skip external URLs + # Resolve the target path relative to the .rels file location + if rels_file.name == ".rels": + # Root .rels file - targets are relative to unpacked_dir + target_path = self.unpacked_dir / target + else: + # Other .rels files - targets are relative to their parent's parent + # e.g., word/_rels/document.xml.rels -> targets relative to word/ + base_dir = rels_dir.parent + target_path = base_dir / target + + # Normalize the path and check if it exists + try: + target_path = target_path.resolve() + if target_path.exists() and target_path.is_file(): + referenced_files.add(target_path) + all_referenced_files.add(target_path) + else: + broken_refs.append((target, rel.sourceline)) + except (OSError, ValueError): + broken_refs.append((target, rel.sourceline)) + + # Report broken references + if broken_refs: + rel_path = rels_file.relative_to(self.unpacked_dir) + for broken_ref, line_num in broken_refs: + errors.append( + f" {rel_path}: Line {line_num}: Broken reference to {broken_ref}" + ) + + except Exception as e: + rel_path = rels_file.relative_to(self.unpacked_dir) + errors.append(f" Error parsing {rel_path}: {e}") + + # Check for unreferenced files (files that exist but are not referenced anywhere) + unreferenced_files = set(all_files) - all_referenced_files + + if unreferenced_files: + for unref_file in sorted(unreferenced_files): + unref_rel_path = unref_file.relative_to(self.unpacked_dir) + errors.append(f" Unreferenced file: {unref_rel_path}") + + if errors: + print(f"FAILED - Found {len(errors)} relationship validation errors:") + for error in errors: + print(error) + print( + "CRITICAL: These errors will cause the document to appear corrupt. " + + "Broken references MUST be fixed, " + + "and unreferenced files MUST be referenced or removed." + ) + return False + else: + if self.verbose: + print( + "PASSED - All references are valid and all files are properly referenced" + ) + return True + + def validate_all_relationship_ids(self): + """ + Validate that all r:id attributes in XML files reference existing IDs + in their corresponding .rels files, and optionally validate relationship types. + """ + import lxml.etree + + errors = [] + + # Process each XML file that might contain r:id references + for xml_file in self.xml_files: + # Skip .rels files themselves + if xml_file.suffix == ".rels": + continue + + # Determine the corresponding .rels file + # For dir/file.xml, it's dir/_rels/file.xml.rels + rels_dir = xml_file.parent / "_rels" + rels_file = rels_dir / f"{xml_file.name}.rels" + + # Skip if there's no corresponding .rels file (that's okay) + if not rels_file.exists(): + continue + + try: + # Parse the .rels file to get valid relationship IDs and their types + rels_root = lxml.etree.parse(str(rels_file)).getroot() + rid_to_type = {} + + for rel in rels_root.findall( + f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship" + ): + rid = rel.get("Id") + rel_type = rel.get("Type", "") + if rid: + # Check for duplicate rIds + if rid in rid_to_type: + rels_rel_path = rels_file.relative_to(self.unpacked_dir) + errors.append( + f" {rels_rel_path}: Line {rel.sourceline}: " + f"Duplicate relationship ID '{rid}' (IDs must be unique)" + ) + # Extract just the type name from the full URL + type_name = ( + rel_type.split("/")[-1] if "/" in rel_type else rel_type + ) + rid_to_type[rid] = type_name + + # Parse the XML file to find all r:id references + xml_root = lxml.etree.parse(str(xml_file)).getroot() + + # Find all elements with r:id attributes + for elem in xml_root.iter(): + # Check for r:id attribute (relationship ID) + rid_attr = elem.get(f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id") + if rid_attr: + xml_rel_path = xml_file.relative_to(self.unpacked_dir) + elem_name = ( + elem.tag.split("}")[-1] if "}" in elem.tag else elem.tag + ) + + # Check if the ID exists + if rid_attr not in rid_to_type: + errors.append( + f" {xml_rel_path}: Line {elem.sourceline}: " + f"<{elem_name}> references non-existent relationship '{rid_attr}' " + f"(valid IDs: {', '.join(sorted(rid_to_type.keys())[:5])}{'...' if len(rid_to_type) > 5 else ''})" + ) + # Check if we have type expectations for this element + elif self.ELEMENT_RELATIONSHIP_TYPES: + expected_type = self._get_expected_relationship_type( + elem_name + ) + if expected_type: + actual_type = rid_to_type[rid_attr] + # Check if the actual type matches or contains the expected type + if expected_type not in actual_type.lower(): + errors.append( + f" {xml_rel_path}: Line {elem.sourceline}: " + f"<{elem_name}> references '{rid_attr}' which points to '{actual_type}' " + f"but should point to a '{expected_type}' relationship" + ) + + except Exception as e: + xml_rel_path = xml_file.relative_to(self.unpacked_dir) + errors.append(f" Error processing {xml_rel_path}: {e}") + + if errors: + print(f"FAILED - Found {len(errors)} relationship ID reference errors:") + for error in errors: + print(error) + print("\nThese ID mismatches will cause the document to appear corrupt!") + return False + else: + if self.verbose: + print("PASSED - All relationship ID references are valid") + return True + + def _get_expected_relationship_type(self, element_name): + """ + Get the expected relationship type for an element. + First checks the explicit mapping, then tries pattern detection. + """ + # Normalize element name to lowercase + elem_lower = element_name.lower() + + # Check explicit mapping first + if elem_lower in self.ELEMENT_RELATIONSHIP_TYPES: + return self.ELEMENT_RELATIONSHIP_TYPES[elem_lower] + + # Try pattern detection for common patterns + # Pattern 1: Elements ending in "Id" often expect a relationship of the prefix type + if elem_lower.endswith("id") and len(elem_lower) > 2: + # e.g., "sldId" -> "sld", "sldMasterId" -> "sldMaster" + prefix = elem_lower[:-2] # Remove "id" + # Check if this might be a compound like "sldMasterId" + if prefix.endswith("master"): + return prefix.lower() + elif prefix.endswith("layout"): + return prefix.lower() + else: + # Simple case like "sldId" -> "slide" + # Common transformations + if prefix == "sld": + return "slide" + return prefix.lower() + + # Pattern 2: Elements ending in "Reference" expect a relationship of the prefix type + if elem_lower.endswith("reference") and len(elem_lower) > 9: + prefix = elem_lower[:-9] # Remove "reference" + return prefix.lower() + + return None + + def validate_content_types(self): + """Validate that all content files are properly declared in [Content_Types].xml.""" + errors = [] + + # Find [Content_Types].xml file + content_types_file = self.unpacked_dir / "[Content_Types].xml" + if not content_types_file.exists(): + print("FAILED - [Content_Types].xml file not found") + return False + + try: + # Parse and get all declared parts and extensions + root = lxml.etree.parse(str(content_types_file)).getroot() + declared_parts = set() + declared_extensions = set() + + # Get Override declarations (specific files) + for override in root.findall( + f".//{{{self.CONTENT_TYPES_NAMESPACE}}}Override" + ): + part_name = override.get("PartName") + if part_name is not None: + declared_parts.add(part_name.lstrip("/")) + + # Get Default declarations (by extension) + for default in root.findall( + f".//{{{self.CONTENT_TYPES_NAMESPACE}}}Default" + ): + extension = default.get("Extension") + if extension is not None: + declared_extensions.add(extension.lower()) + + # Root elements that require content type declaration + declarable_roots = { + "sld", + "sldLayout", + "sldMaster", + "presentation", # PowerPoint + "document", # Word + "workbook", + "worksheet", # Excel + "theme", # Common + } + + # Common media file extensions that should be declared + media_extensions = { + "png": "image/png", + "jpg": "image/jpeg", + "jpeg": "image/jpeg", + "gif": "image/gif", + "bmp": "image/bmp", + "tiff": "image/tiff", + "wmf": "image/x-wmf", + "emf": "image/x-emf", + } + + # Get all files in the unpacked directory + all_files = list(self.unpacked_dir.rglob("*")) + all_files = [f for f in all_files if f.is_file()] + + # Check all XML files for Override declarations + for xml_file in self.xml_files: + path_str = str(xml_file.relative_to(self.unpacked_dir)).replace( + "\\", "/" + ) + + # Skip non-content files + if any( + skip in path_str + for skip in [".rels", "[Content_Types]", "docProps/", "_rels/"] + ): + continue + + try: + root_tag = lxml.etree.parse(str(xml_file)).getroot().tag + root_name = root_tag.split("}")[-1] if "}" in root_tag else root_tag + + if root_name in declarable_roots and path_str not in declared_parts: + errors.append( + f" {path_str}: File with <{root_name}> root not declared in [Content_Types].xml" + ) + + except Exception: + continue # Skip unparseable files + + # Check all non-XML files for Default extension declarations + for file_path in all_files: + # Skip XML files and metadata files (already checked above) + if file_path.suffix.lower() in {".xml", ".rels"}: + continue + if file_path.name == "[Content_Types].xml": + continue + if "_rels" in file_path.parts or "docProps" in file_path.parts: + continue + + extension = file_path.suffix.lstrip(".").lower() + if extension and extension not in declared_extensions: + # Check if it's a known media extension that should be declared + if extension in media_extensions: + relative_path = file_path.relative_to(self.unpacked_dir) + errors.append( + f' {relative_path}: File with extension \'{extension}\' not declared in [Content_Types].xml - should add: ' + ) + + except Exception as e: + errors.append(f" Error parsing [Content_Types].xml: {e}") + + if errors: + print(f"FAILED - Found {len(errors)} content type declaration errors:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print( + "PASSED - All content files are properly declared in [Content_Types].xml" + ) + return True + + def validate_file_against_xsd(self, xml_file, verbose=False): + """Validate a single XML file against XSD schema, comparing with original. + + Args: + xml_file: Path to XML file to validate + verbose: Enable verbose output + + Returns: + tuple: (is_valid, new_errors_set) where is_valid is True/False/None (skipped) + """ + # Resolve both paths to handle symlinks + xml_file = Path(xml_file).resolve() + unpacked_dir = self.unpacked_dir.resolve() + + # Validate current file + is_valid, current_errors = self._validate_single_file_xsd( + xml_file, unpacked_dir + ) + + if is_valid is None: + return None, set() # Skipped + elif is_valid: + return True, set() # Valid, no errors + + # Get errors from original file for this specific file + original_errors = self._get_original_file_errors(xml_file) + + # Compare with original (both are guaranteed to be sets here) + assert current_errors is not None + new_errors = current_errors - original_errors + + if new_errors: + if verbose: + relative_path = xml_file.relative_to(unpacked_dir) + print(f"FAILED - {relative_path}: {len(new_errors)} new error(s)") + for error in list(new_errors)[:3]: + truncated = error[:250] + "..." if len(error) > 250 else error + print(f" - {truncated}") + return False, new_errors + else: + # All errors existed in original + if verbose: + print( + f"PASSED - No new errors (original had {len(current_errors)} errors)" + ) + return True, set() + + def validate_against_xsd(self): + """Validate XML files against XSD schemas, showing only new errors compared to original.""" + new_errors = [] + original_error_count = 0 + valid_count = 0 + skipped_count = 0 + + for xml_file in self.xml_files: + relative_path = str(xml_file.relative_to(self.unpacked_dir)) + is_valid, new_file_errors = self.validate_file_against_xsd( + xml_file, verbose=False + ) + + if is_valid is None: + skipped_count += 1 + continue + elif is_valid and not new_file_errors: + valid_count += 1 + continue + elif is_valid: + # Had errors but all existed in original + original_error_count += 1 + valid_count += 1 + continue + + # Has new errors + new_errors.append(f" {relative_path}: {len(new_file_errors)} new error(s)") + for error in list(new_file_errors)[:3]: # Show first 3 errors + new_errors.append( + f" - {error[:250]}..." if len(error) > 250 else f" - {error}" + ) + + # Print summary + if self.verbose: + print(f"Validated {len(self.xml_files)} files:") + print(f" - Valid: {valid_count}") + print(f" - Skipped (no schema): {skipped_count}") + if original_error_count: + print(f" - With original errors (ignored): {original_error_count}") + print( + f" - With NEW errors: {len(new_errors) > 0 and len([e for e in new_errors if not e.startswith(' ')]) or 0}" + ) + + if new_errors: + print("\nFAILED - Found NEW validation errors:") + for error in new_errors: + print(error) + return False + else: + if self.verbose: + print("\nPASSED - No new XSD validation errors introduced") + return True + + def _get_schema_path(self, xml_file): + """Determine the appropriate schema path for an XML file.""" + # Check exact filename match + if xml_file.name in self.SCHEMA_MAPPINGS: + return self.schemas_dir / self.SCHEMA_MAPPINGS[xml_file.name] + + # Check .rels files + if xml_file.suffix == ".rels": + return self.schemas_dir / self.SCHEMA_MAPPINGS[".rels"] + + # Check chart files + if "charts/" in str(xml_file) and xml_file.name.startswith("chart"): + return self.schemas_dir / self.SCHEMA_MAPPINGS["chart"] + + # Check theme files + if "theme/" in str(xml_file) and xml_file.name.startswith("theme"): + return self.schemas_dir / self.SCHEMA_MAPPINGS["theme"] + + # Check if file is in a main content folder and use appropriate schema + if xml_file.parent.name in self.MAIN_CONTENT_FOLDERS: + return self.schemas_dir / self.SCHEMA_MAPPINGS[xml_file.parent.name] + + return None + + def _clean_ignorable_namespaces(self, xml_doc): + """Remove attributes and elements not in allowed namespaces.""" + # Create a clean copy + xml_string = lxml.etree.tostring(xml_doc, encoding="unicode") + xml_copy = lxml.etree.fromstring(xml_string) + + # Remove attributes not in allowed namespaces + for elem in xml_copy.iter(): + attrs_to_remove = [] + + for attr in elem.attrib: + # Check if attribute is from a namespace other than allowed ones + if "{" in attr: + ns = attr.split("}")[0][1:] + if ns not in self.OOXML_NAMESPACES: + attrs_to_remove.append(attr) + + # Remove collected attributes + for attr in attrs_to_remove: + del elem.attrib[attr] + + # Remove elements not in allowed namespaces + self._remove_ignorable_elements(xml_copy) + + return lxml.etree.ElementTree(xml_copy) + + def _remove_ignorable_elements(self, root): + """Recursively remove all elements not in allowed namespaces.""" + elements_to_remove = [] + + # Find elements to remove + for elem in list(root): + # Skip non-element nodes (comments, processing instructions, etc.) + if not hasattr(elem, "tag") or callable(elem.tag): + continue + + tag_str = str(elem.tag) + if tag_str.startswith("{"): + ns = tag_str.split("}")[0][1:] + if ns not in self.OOXML_NAMESPACES: + elements_to_remove.append(elem) + continue + + # Recursively clean child elements + self._remove_ignorable_elements(elem) + + # Remove collected elements + for elem in elements_to_remove: + root.remove(elem) + + def _preprocess_for_mc_ignorable(self, xml_doc): + """Preprocess XML to handle mc:Ignorable attribute properly.""" + # Remove mc:Ignorable attributes before validation + root = xml_doc.getroot() + + # Remove mc:Ignorable attribute from root + if f"{{{self.MC_NAMESPACE}}}Ignorable" in root.attrib: + del root.attrib[f"{{{self.MC_NAMESPACE}}}Ignorable"] + + return xml_doc + + def _validate_single_file_xsd(self, xml_file, base_path): + """Validate a single XML file against XSD schema. Returns (is_valid, errors_set).""" + schema_path = self._get_schema_path(xml_file) + if not schema_path: + return None, None # Skip file + + try: + # Load schema + with open(schema_path, "rb") as xsd_file: + parser = lxml.etree.XMLParser() + xsd_doc = lxml.etree.parse( + xsd_file, parser=parser, base_url=str(schema_path) + ) + schema = lxml.etree.XMLSchema(xsd_doc) + + # Load and preprocess XML + with open(xml_file, "r") as f: + xml_doc = lxml.etree.parse(f) + + xml_doc, _ = self._remove_template_tags_from_text_nodes(xml_doc) + xml_doc = self._preprocess_for_mc_ignorable(xml_doc) + + # Clean ignorable namespaces if needed + relative_path = xml_file.relative_to(base_path) + if ( + relative_path.parts + and relative_path.parts[0] in self.MAIN_CONTENT_FOLDERS + ): + xml_doc = self._clean_ignorable_namespaces(xml_doc) + + # Validate + if schema.validate(xml_doc): + return True, set() + else: + errors = set() + for error in schema.error_log: + # Store normalized error message (without line numbers for comparison) + errors.add(error.message) + return False, errors + + except Exception as e: + return False, {str(e)} + + def _get_original_file_errors(self, xml_file): + """Get XSD validation errors from a single file in the original document. + + Args: + xml_file: Path to the XML file in unpacked_dir to check + + Returns: + set: Set of error messages from the original file + """ + import tempfile + import zipfile + + # Resolve both paths to handle symlinks (e.g., /var vs /private/var on macOS) + xml_file = Path(xml_file).resolve() + unpacked_dir = self.unpacked_dir.resolve() + relative_path = xml_file.relative_to(unpacked_dir) + + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Extract original file + with zipfile.ZipFile(self.original_file, "r") as zip_ref: + zip_ref.extractall(temp_path) + + # Find corresponding file in original + original_xml_file = temp_path / relative_path + + if not original_xml_file.exists(): + # File didn't exist in original, so no original errors + return set() + + # Validate the specific file in original + is_valid, errors = self._validate_single_file_xsd( + original_xml_file, temp_path + ) + return errors if errors else set() + + def _remove_template_tags_from_text_nodes(self, xml_doc): + """Remove template tags from XML text nodes and collect warnings. + + Template tags follow the pattern {{ ... }} and are used as placeholders + for content replacement. They should be removed from text content before + XSD validation while preserving XML structure. + + Returns: + tuple: (cleaned_xml_doc, warnings_list) + """ + warnings = [] + template_pattern = re.compile(r"\{\{[^}]*\}\}") + + # Create a copy of the document to avoid modifying the original + xml_string = lxml.etree.tostring(xml_doc, encoding="unicode") + xml_copy = lxml.etree.fromstring(xml_string) + + def process_text_content(text, content_type): + if not text: + return text + matches = list(template_pattern.finditer(text)) + if matches: + for match in matches: + warnings.append( + f"Found template tag in {content_type}: {match.group()}" + ) + return template_pattern.sub("", text) + return text + + # Process all text nodes in the document + for elem in xml_copy.iter(): + # Skip processing if this is a w:t element + if not hasattr(elem, "tag") or callable(elem.tag): + continue + tag_str = str(elem.tag) + if tag_str.endswith("}t") or tag_str == "t": + continue + + elem.text = process_text_content(elem.text, "text content") + elem.tail = process_text_content(elem.tail, "tail content") + + return lxml.etree.ElementTree(xml_copy), warnings + + +if __name__ == "__main__": + raise RuntimeError("This module should not be run directly.") diff --git a/skills/document-skills/docx/ooxml/scripts/validation/docx.py b/skills/document-skills/docx/ooxml/scripts/validation/docx.py new file mode 100644 index 0000000..602c470 --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/validation/docx.py @@ -0,0 +1,274 @@ +""" +Validator for Word document XML files against XSD schemas. +""" + +import re +import tempfile +import zipfile + +import lxml.etree + +from .base import BaseSchemaValidator + + +class DOCXSchemaValidator(BaseSchemaValidator): + """Validator for Word document XML files against XSD schemas.""" + + # Word-specific namespace + WORD_2006_NAMESPACE = "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + + # Word-specific element to relationship type mappings + # Start with empty mapping - add specific cases as we discover them + ELEMENT_RELATIONSHIP_TYPES = {} + + def validate(self): + """Run all validation checks and return True if all pass.""" + # Test 0: XML well-formedness + if not self.validate_xml(): + return False + + # Test 1: Namespace declarations + all_valid = True + if not self.validate_namespaces(): + all_valid = False + + # Test 2: Unique IDs + if not self.validate_unique_ids(): + all_valid = False + + # Test 3: Relationship and file reference validation + if not self.validate_file_references(): + all_valid = False + + # Test 4: Content type declarations + if not self.validate_content_types(): + all_valid = False + + # Test 5: XSD schema validation + if not self.validate_against_xsd(): + all_valid = False + + # Test 6: Whitespace preservation + if not self.validate_whitespace_preservation(): + all_valid = False + + # Test 7: Deletion validation + if not self.validate_deletions(): + all_valid = False + + # Test 8: Insertion validation + if not self.validate_insertions(): + all_valid = False + + # Test 9: Relationship ID reference validation + if not self.validate_all_relationship_ids(): + all_valid = False + + # Count and compare paragraphs + self.compare_paragraph_counts() + + return all_valid + + def validate_whitespace_preservation(self): + """ + Validate that w:t elements with whitespace have xml:space='preserve'. + """ + errors = [] + + for xml_file in self.xml_files: + # Only check document.xml files + if xml_file.name != "document.xml": + continue + + try: + root = lxml.etree.parse(str(xml_file)).getroot() + + # Find all w:t elements + for elem in root.iter(f"{{{self.WORD_2006_NAMESPACE}}}t"): + if elem.text: + text = elem.text + # Check if text starts or ends with whitespace + if re.match(r"^\s.*", text) or re.match(r".*\s$", text): + # Check if xml:space="preserve" attribute exists + xml_space_attr = f"{{{self.XML_NAMESPACE}}}space" + if ( + xml_space_attr not in elem.attrib + or elem.attrib[xml_space_attr] != "preserve" + ): + # Show a preview of the text + text_preview = ( + repr(text)[:50] + "..." + if len(repr(text)) > 50 + else repr(text) + ) + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {elem.sourceline}: w:t element with whitespace missing xml:space='preserve': {text_preview}" + ) + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} whitespace preservation violations:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - All whitespace is properly preserved") + return True + + def validate_deletions(self): + """ + Validate that w:t elements are not within w:del elements. + For some reason, XSD validation does not catch this, so we do it manually. + """ + errors = [] + + for xml_file in self.xml_files: + # Only check document.xml files + if xml_file.name != "document.xml": + continue + + try: + root = lxml.etree.parse(str(xml_file)).getroot() + + # Find all w:t elements that are descendants of w:del elements + namespaces = {"w": self.WORD_2006_NAMESPACE} + xpath_expression = ".//w:del//w:t" + problematic_t_elements = root.xpath( + xpath_expression, namespaces=namespaces + ) + for t_elem in problematic_t_elements: + if t_elem.text: + # Show a preview of the text + text_preview = ( + repr(t_elem.text)[:50] + "..." + if len(repr(t_elem.text)) > 50 + else repr(t_elem.text) + ) + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {t_elem.sourceline}: found within : {text_preview}" + ) + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} deletion validation violations:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - No w:t elements found within w:del elements") + return True + + def count_paragraphs_in_unpacked(self): + """Count the number of paragraphs in the unpacked document.""" + count = 0 + + for xml_file in self.xml_files: + # Only check document.xml files + if xml_file.name != "document.xml": + continue + + try: + root = lxml.etree.parse(str(xml_file)).getroot() + # Count all w:p elements + paragraphs = root.findall(f".//{{{self.WORD_2006_NAMESPACE}}}p") + count = len(paragraphs) + except Exception as e: + print(f"Error counting paragraphs in unpacked document: {e}") + + return count + + def count_paragraphs_in_original(self): + """Count the number of paragraphs in the original docx file.""" + count = 0 + + try: + # Create temporary directory to unpack original + with tempfile.TemporaryDirectory() as temp_dir: + # Unpack original docx + with zipfile.ZipFile(self.original_file, "r") as zip_ref: + zip_ref.extractall(temp_dir) + + # Parse document.xml + doc_xml_path = temp_dir + "/word/document.xml" + root = lxml.etree.parse(doc_xml_path).getroot() + + # Count all w:p elements + paragraphs = root.findall(f".//{{{self.WORD_2006_NAMESPACE}}}p") + count = len(paragraphs) + + except Exception as e: + print(f"Error counting paragraphs in original document: {e}") + + return count + + def validate_insertions(self): + """ + Validate that w:delText elements are not within w:ins elements. + w:delText is only allowed in w:ins if nested within a w:del. + """ + errors = [] + + for xml_file in self.xml_files: + if xml_file.name != "document.xml": + continue + + try: + root = lxml.etree.parse(str(xml_file)).getroot() + namespaces = {"w": self.WORD_2006_NAMESPACE} + + # Find w:delText in w:ins that are NOT within w:del + invalid_elements = root.xpath( + ".//w:ins//w:delText[not(ancestor::w:del)]", + namespaces=namespaces + ) + + for elem in invalid_elements: + text_preview = ( + repr(elem.text or "")[:50] + "..." + if len(repr(elem.text or "")) > 50 + else repr(elem.text or "") + ) + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {elem.sourceline}: within : {text_preview}" + ) + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} insertion validation violations:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - No w:delText elements within w:ins elements") + return True + + def compare_paragraph_counts(self): + """Compare paragraph counts between original and new document.""" + original_count = self.count_paragraphs_in_original() + new_count = self.count_paragraphs_in_unpacked() + + diff = new_count - original_count + diff_str = f"+{diff}" if diff > 0 else str(diff) + print(f"\nParagraphs: {original_count} → {new_count} ({diff_str})") + + +if __name__ == "__main__": + raise RuntimeError("This module should not be run directly.") diff --git a/skills/document-skills/docx/ooxml/scripts/validation/pptx.py b/skills/document-skills/docx/ooxml/scripts/validation/pptx.py new file mode 100644 index 0000000..66d5b1e --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/validation/pptx.py @@ -0,0 +1,315 @@ +""" +Validator for PowerPoint presentation XML files against XSD schemas. +""" + +import re + +from .base import BaseSchemaValidator + + +class PPTXSchemaValidator(BaseSchemaValidator): + """Validator for PowerPoint presentation XML files against XSD schemas.""" + + # PowerPoint presentation namespace + PRESENTATIONML_NAMESPACE = ( + "http://schemas.openxmlformats.org/presentationml/2006/main" + ) + + # PowerPoint-specific element to relationship type mappings + ELEMENT_RELATIONSHIP_TYPES = { + "sldid": "slide", + "sldmasterid": "slidemaster", + "notesmasterid": "notesmaster", + "sldlayoutid": "slidelayout", + "themeid": "theme", + "tablestyleid": "tablestyles", + } + + def validate(self): + """Run all validation checks and return True if all pass.""" + # Test 0: XML well-formedness + if not self.validate_xml(): + return False + + # Test 1: Namespace declarations + all_valid = True + if not self.validate_namespaces(): + all_valid = False + + # Test 2: Unique IDs + if not self.validate_unique_ids(): + all_valid = False + + # Test 3: UUID ID validation + if not self.validate_uuid_ids(): + all_valid = False + + # Test 4: Relationship and file reference validation + if not self.validate_file_references(): + all_valid = False + + # Test 5: Slide layout ID validation + if not self.validate_slide_layout_ids(): + all_valid = False + + # Test 6: Content type declarations + if not self.validate_content_types(): + all_valid = False + + # Test 7: XSD schema validation + if not self.validate_against_xsd(): + all_valid = False + + # Test 8: Notes slide reference validation + if not self.validate_notes_slide_references(): + all_valid = False + + # Test 9: Relationship ID reference validation + if not self.validate_all_relationship_ids(): + all_valid = False + + # Test 10: Duplicate slide layout references validation + if not self.validate_no_duplicate_slide_layouts(): + all_valid = False + + return all_valid + + def validate_uuid_ids(self): + """Validate that ID attributes that look like UUIDs contain only hex values.""" + import lxml.etree + + errors = [] + # UUID pattern: 8-4-4-4-12 hex digits with optional braces/hyphens + uuid_pattern = re.compile( + r"^[\{\(]?[0-9A-Fa-f]{8}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{4}-?[0-9A-Fa-f]{12}[\}\)]?$" + ) + + for xml_file in self.xml_files: + try: + root = lxml.etree.parse(str(xml_file)).getroot() + + # Check all elements for ID attributes + for elem in root.iter(): + for attr, value in elem.attrib.items(): + # Check if this is an ID attribute + attr_name = attr.split("}")[-1].lower() + if attr_name == "id" or attr_name.endswith("id"): + # Check if value looks like a UUID (has the right length and pattern structure) + if self._looks_like_uuid(value): + # Validate that it contains only hex characters in the right positions + if not uuid_pattern.match(value): + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: " + f"Line {elem.sourceline}: ID '{value}' appears to be a UUID but contains invalid hex characters" + ) + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {xml_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} UUID ID validation errors:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - All UUID-like IDs contain valid hex values") + return True + + def _looks_like_uuid(self, value): + """Check if a value has the general structure of a UUID.""" + # Remove common UUID delimiters + clean_value = value.strip("{}()").replace("-", "") + # Check if it's 32 hex-like characters (could include invalid hex chars) + return len(clean_value) == 32 and all(c.isalnum() for c in clean_value) + + def validate_slide_layout_ids(self): + """Validate that sldLayoutId elements in slide masters reference valid slide layouts.""" + import lxml.etree + + errors = [] + + # Find all slide master files + slide_masters = list(self.unpacked_dir.glob("ppt/slideMasters/*.xml")) + + if not slide_masters: + if self.verbose: + print("PASSED - No slide masters found") + return True + + for slide_master in slide_masters: + try: + # Parse the slide master file + root = lxml.etree.parse(str(slide_master)).getroot() + + # Find the corresponding _rels file for this slide master + rels_file = slide_master.parent / "_rels" / f"{slide_master.name}.rels" + + if not rels_file.exists(): + errors.append( + f" {slide_master.relative_to(self.unpacked_dir)}: " + f"Missing relationships file: {rels_file.relative_to(self.unpacked_dir)}" + ) + continue + + # Parse the relationships file + rels_root = lxml.etree.parse(str(rels_file)).getroot() + + # Build a set of valid relationship IDs that point to slide layouts + valid_layout_rids = set() + for rel in rels_root.findall( + f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship" + ): + rel_type = rel.get("Type", "") + if "slideLayout" in rel_type: + valid_layout_rids.add(rel.get("Id")) + + # Find all sldLayoutId elements in the slide master + for sld_layout_id in root.findall( + f".//{{{self.PRESENTATIONML_NAMESPACE}}}sldLayoutId" + ): + r_id = sld_layout_id.get( + f"{{{self.OFFICE_RELATIONSHIPS_NAMESPACE}}}id" + ) + layout_id = sld_layout_id.get("id") + + if r_id and r_id not in valid_layout_rids: + errors.append( + f" {slide_master.relative_to(self.unpacked_dir)}: " + f"Line {sld_layout_id.sourceline}: sldLayoutId with id='{layout_id}' " + f"references r:id='{r_id}' which is not found in slide layout relationships" + ) + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {slide_master.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print(f"FAILED - Found {len(errors)} slide layout ID validation errors:") + for error in errors: + print(error) + print( + "Remove invalid references or add missing slide layouts to the relationships file." + ) + return False + else: + if self.verbose: + print("PASSED - All slide layout IDs reference valid slide layouts") + return True + + def validate_no_duplicate_slide_layouts(self): + """Validate that each slide has exactly one slideLayout reference.""" + import lxml.etree + + errors = [] + slide_rels_files = list(self.unpacked_dir.glob("ppt/slides/_rels/*.xml.rels")) + + for rels_file in slide_rels_files: + try: + root = lxml.etree.parse(str(rels_file)).getroot() + + # Find all slideLayout relationships + layout_rels = [ + rel + for rel in root.findall( + f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship" + ) + if "slideLayout" in rel.get("Type", "") + ] + + if len(layout_rels) > 1: + errors.append( + f" {rels_file.relative_to(self.unpacked_dir)}: has {len(layout_rels)} slideLayout references" + ) + + except Exception as e: + errors.append( + f" {rels_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + if errors: + print("FAILED - Found slides with duplicate slideLayout references:") + for error in errors: + print(error) + return False + else: + if self.verbose: + print("PASSED - All slides have exactly one slideLayout reference") + return True + + def validate_notes_slide_references(self): + """Validate that each notesSlide file is referenced by only one slide.""" + import lxml.etree + + errors = [] + notes_slide_references = {} # Track which slides reference each notesSlide + + # Find all slide relationship files + slide_rels_files = list(self.unpacked_dir.glob("ppt/slides/_rels/*.xml.rels")) + + if not slide_rels_files: + if self.verbose: + print("PASSED - No slide relationship files found") + return True + + for rels_file in slide_rels_files: + try: + # Parse the relationships file + root = lxml.etree.parse(str(rels_file)).getroot() + + # Find all notesSlide relationships + for rel in root.findall( + f".//{{{self.PACKAGE_RELATIONSHIPS_NAMESPACE}}}Relationship" + ): + rel_type = rel.get("Type", "") + if "notesSlide" in rel_type: + target = rel.get("Target", "") + if target: + # Normalize the target path to handle relative paths + normalized_target = target.replace("../", "") + + # Track which slide references this notesSlide + slide_name = rels_file.stem.replace( + ".xml", "" + ) # e.g., "slide1" + + if normalized_target not in notes_slide_references: + notes_slide_references[normalized_target] = [] + notes_slide_references[normalized_target].append( + (slide_name, rels_file) + ) + + except (lxml.etree.XMLSyntaxError, Exception) as e: + errors.append( + f" {rels_file.relative_to(self.unpacked_dir)}: Error: {e}" + ) + + # Check for duplicate references + for target, references in notes_slide_references.items(): + if len(references) > 1: + slide_names = [ref[0] for ref in references] + errors.append( + f" Notes slide '{target}' is referenced by multiple slides: {', '.join(slide_names)}" + ) + for slide_name, rels_file in references: + errors.append(f" - {rels_file.relative_to(self.unpacked_dir)}") + + if errors: + print( + f"FAILED - Found {len([e for e in errors if not e.startswith(' ')])} notes slide reference validation errors:" + ) + for error in errors: + print(error) + print("Each slide may optionally have its own slide file.") + return False + else: + if self.verbose: + print("PASSED - All notes slide references are unique") + return True + + +if __name__ == "__main__": + raise RuntimeError("This module should not be run directly.") diff --git a/skills/document-skills/docx/ooxml/scripts/validation/redlining.py b/skills/document-skills/docx/ooxml/scripts/validation/redlining.py new file mode 100644 index 0000000..7ed425e --- /dev/null +++ b/skills/document-skills/docx/ooxml/scripts/validation/redlining.py @@ -0,0 +1,279 @@ +""" +Validator for tracked changes in Word documents. +""" + +import subprocess +import tempfile +import zipfile +from pathlib import Path + + +class RedliningValidator: + """Validator for tracked changes in Word documents.""" + + def __init__(self, unpacked_dir, original_docx, verbose=False): + self.unpacked_dir = Path(unpacked_dir) + self.original_docx = Path(original_docx) + self.verbose = verbose + self.namespaces = { + "w": "http://schemas.openxmlformats.org/wordprocessingml/2006/main" + } + + def validate(self): + """Main validation method that returns True if valid, False otherwise.""" + # Verify unpacked directory exists and has correct structure + modified_file = self.unpacked_dir / "word" / "document.xml" + if not modified_file.exists(): + print(f"FAILED - Modified document.xml not found at {modified_file}") + return False + + # First, check if there are any tracked changes by Claude to validate + try: + import xml.etree.ElementTree as ET + + tree = ET.parse(modified_file) + root = tree.getroot() + + # Check for w:del or w:ins tags authored by Claude + del_elements = root.findall(".//w:del", self.namespaces) + ins_elements = root.findall(".//w:ins", self.namespaces) + + # Filter to only include changes by Claude + claude_del_elements = [ + elem + for elem in del_elements + if elem.get(f"{{{self.namespaces['w']}}}author") == "Claude" + ] + claude_ins_elements = [ + elem + for elem in ins_elements + if elem.get(f"{{{self.namespaces['w']}}}author") == "Claude" + ] + + # Redlining validation is only needed if tracked changes by Claude have been used. + if not claude_del_elements and not claude_ins_elements: + if self.verbose: + print("PASSED - No tracked changes by Claude found.") + return True + + except Exception: + # If we can't parse the XML, continue with full validation + pass + + # Create temporary directory for unpacking original docx + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Unpack original docx + try: + with zipfile.ZipFile(self.original_docx, "r") as zip_ref: + zip_ref.extractall(temp_path) + except Exception as e: + print(f"FAILED - Error unpacking original docx: {e}") + return False + + original_file = temp_path / "word" / "document.xml" + if not original_file.exists(): + print( + f"FAILED - Original document.xml not found in {self.original_docx}" + ) + return False + + # Parse both XML files using xml.etree.ElementTree for redlining validation + try: + import xml.etree.ElementTree as ET + + modified_tree = ET.parse(modified_file) + modified_root = modified_tree.getroot() + original_tree = ET.parse(original_file) + original_root = original_tree.getroot() + except ET.ParseError as e: + print(f"FAILED - Error parsing XML files: {e}") + return False + + # Remove Claude's tracked changes from both documents + self._remove_claude_tracked_changes(original_root) + self._remove_claude_tracked_changes(modified_root) + + # Extract and compare text content + modified_text = self._extract_text_content(modified_root) + original_text = self._extract_text_content(original_root) + + if modified_text != original_text: + # Show detailed character-level differences for each paragraph + error_message = self._generate_detailed_diff( + original_text, modified_text + ) + print(error_message) + return False + + if self.verbose: + print("PASSED - All changes by Claude are properly tracked") + return True + + def _generate_detailed_diff(self, original_text, modified_text): + """Generate detailed word-level differences using git word diff.""" + error_parts = [ + "FAILED - Document text doesn't match after removing Claude's tracked changes", + "", + "Likely causes:", + " 1. Modified text inside another author's or tags", + " 2. Made edits without proper tracked changes", + " 3. Didn't nest inside when deleting another's insertion", + "", + "For pre-redlined documents, use correct patterns:", + " - To reject another's INSERTION: Nest inside their ", + " - To restore another's DELETION: Add new AFTER their ", + "", + ] + + # Show git word diff + git_diff = self._get_git_word_diff(original_text, modified_text) + if git_diff: + error_parts.extend(["Differences:", "============", git_diff]) + else: + error_parts.append("Unable to generate word diff (git not available)") + + return "\n".join(error_parts) + + def _get_git_word_diff(self, original_text, modified_text): + """Generate word diff using git with character-level precision.""" + try: + with tempfile.TemporaryDirectory() as temp_dir: + temp_path = Path(temp_dir) + + # Create two files + original_file = temp_path / "original.txt" + modified_file = temp_path / "modified.txt" + + original_file.write_text(original_text, encoding="utf-8") + modified_file.write_text(modified_text, encoding="utf-8") + + # Try character-level diff first for precise differences + result = subprocess.run( + [ + "git", + "diff", + "--word-diff=plain", + "--word-diff-regex=.", # Character-by-character diff + "-U0", # Zero lines of context - show only changed lines + "--no-index", + str(original_file), + str(modified_file), + ], + capture_output=True, + text=True, + ) + + if result.stdout.strip(): + # Clean up the output - remove git diff header lines + lines = result.stdout.split("\n") + # Skip the header lines (diff --git, index, +++, ---, @@) + content_lines = [] + in_content = False + for line in lines: + if line.startswith("@@"): + in_content = True + continue + if in_content and line.strip(): + content_lines.append(line) + + if content_lines: + return "\n".join(content_lines) + + # Fallback to word-level diff if character-level is too verbose + result = subprocess.run( + [ + "git", + "diff", + "--word-diff=plain", + "-U0", # Zero lines of context + "--no-index", + str(original_file), + str(modified_file), + ], + capture_output=True, + text=True, + ) + + if result.stdout.strip(): + lines = result.stdout.split("\n") + content_lines = [] + in_content = False + for line in lines: + if line.startswith("@@"): + in_content = True + continue + if in_content and line.strip(): + content_lines.append(line) + return "\n".join(content_lines) + + except (subprocess.CalledProcessError, FileNotFoundError, Exception): + # Git not available or other error, return None to use fallback + pass + + return None + + def _remove_claude_tracked_changes(self, root): + """Remove tracked changes authored by Claude from the XML root.""" + ins_tag = f"{{{self.namespaces['w']}}}ins" + del_tag = f"{{{self.namespaces['w']}}}del" + author_attr = f"{{{self.namespaces['w']}}}author" + + # Remove w:ins elements + for parent in root.iter(): + to_remove = [] + for child in parent: + if child.tag == ins_tag and child.get(author_attr) == "Claude": + to_remove.append(child) + for elem in to_remove: + parent.remove(elem) + + # Unwrap content in w:del elements where author is "Claude" + deltext_tag = f"{{{self.namespaces['w']}}}delText" + t_tag = f"{{{self.namespaces['w']}}}t" + + for parent in root.iter(): + to_process = [] + for child in parent: + if child.tag == del_tag and child.get(author_attr) == "Claude": + to_process.append((child, list(parent).index(child))) + + # Process in reverse order to maintain indices + for del_elem, del_index in reversed(to_process): + # Convert w:delText to w:t before moving + for elem in del_elem.iter(): + if elem.tag == deltext_tag: + elem.tag = t_tag + + # Move all children of w:del to its parent before removing w:del + for child in reversed(list(del_elem)): + parent.insert(del_index, child) + parent.remove(del_elem) + + def _extract_text_content(self, root): + """Extract text content from Word XML, preserving paragraph structure. + + Empty paragraphs are skipped to avoid false positives when tracked + insertions add only structural elements without text content. + """ + p_tag = f"{{{self.namespaces['w']}}}p" + t_tag = f"{{{self.namespaces['w']}}}t" + + paragraphs = [] + for p_elem in root.findall(f".//{p_tag}"): + # Get all text elements within this paragraph + text_parts = [] + for t_elem in p_elem.findall(f".//{t_tag}"): + if t_elem.text: + text_parts.append(t_elem.text) + paragraph_text = "".join(text_parts) + # Skip empty paragraphs - they don't affect content validation + if paragraph_text: + paragraphs.append(paragraph_text) + + return "\n".join(paragraphs) + + +if __name__ == "__main__": + raise RuntimeError("This module should not be run directly.") diff --git a/skills/document-skills/docx/scripts/__init__.py b/skills/document-skills/docx/scripts/__init__.py new file mode 100755 index 0000000..bf9c562 --- /dev/null +++ b/skills/document-skills/docx/scripts/__init__.py @@ -0,0 +1 @@ +# Make scripts directory a package for relative imports in tests diff --git a/skills/document-skills/docx/scripts/document.py b/skills/document-skills/docx/scripts/document.py new file mode 100755 index 0000000..ae9328d --- /dev/null +++ b/skills/document-skills/docx/scripts/document.py @@ -0,0 +1,1276 @@ +#!/usr/bin/env python3 +""" +Library for working with Word documents: comments, tracked changes, and editing. + +Usage: + from skills.docx.scripts.document import Document + + # Initialize + doc = Document('workspace/unpacked') + doc = Document('workspace/unpacked', author="John Doe", initials="JD") + + # Find nodes + node = doc["word/document.xml"].get_node(tag="w:del", attrs={"w:id": "1"}) + node = doc["word/document.xml"].get_node(tag="w:p", line_number=10) + + # Add comments + doc.add_comment(start=node, end=node, text="Comment text") + doc.reply_to_comment(parent_comment_id=0, text="Reply text") + + # Suggest tracked changes + doc["word/document.xml"].suggest_deletion(node) # Delete content + doc["word/document.xml"].revert_insertion(ins_node) # Reject insertion + doc["word/document.xml"].revert_deletion(del_node) # Reject deletion + + # Save + doc.save() +""" + +import html +import random +import shutil +import tempfile +from datetime import datetime, timezone +from pathlib import Path + +from defusedxml import minidom +from ooxml.scripts.pack import pack_document +from ooxml.scripts.validation.docx import DOCXSchemaValidator +from ooxml.scripts.validation.redlining import RedliningValidator + +from .utilities import XMLEditor + +# Path to template files +TEMPLATE_DIR = Path(__file__).parent / "templates" + + +class DocxXMLEditor(XMLEditor): + """XMLEditor that automatically applies RSID, author, and date to new elements. + + Automatically adds attributes to elements that support them when inserting new content: + - w:rsidR, w:rsidRDefault, w:rsidP (for w:p and w:r elements) + - w:author and w:date (for w:ins, w:del, w:comment elements) + - w:id (for w:ins and w:del elements) + + Attributes: + dom (defusedxml.minidom.Document): The DOM document for direct manipulation + """ + + def __init__( + self, xml_path, rsid: str, author: str = "Claude", initials: str = "C" + ): + """Initialize with required RSID and optional author. + + Args: + xml_path: Path to XML file to edit + rsid: RSID to automatically apply to new elements + author: Author name for tracked changes and comments (default: "Claude") + initials: Author initials (default: "C") + """ + super().__init__(xml_path) + self.rsid = rsid + self.author = author + self.initials = initials + + def _get_next_change_id(self): + """Get the next available change ID by checking all tracked change elements.""" + max_id = -1 + for tag in ("w:ins", "w:del"): + elements = self.dom.getElementsByTagName(tag) + for elem in elements: + change_id = elem.getAttribute("w:id") + if change_id: + try: + max_id = max(max_id, int(change_id)) + except ValueError: + pass + return max_id + 1 + + def _ensure_w16du_namespace(self): + """Ensure w16du namespace is declared on the root element.""" + root = self.dom.documentElement + if not root.hasAttribute("xmlns:w16du"): # type: ignore + root.setAttribute( # type: ignore + "xmlns:w16du", + "http://schemas.microsoft.com/office/word/2023/wordml/word16du", + ) + + def _ensure_w16cex_namespace(self): + """Ensure w16cex namespace is declared on the root element.""" + root = self.dom.documentElement + if not root.hasAttribute("xmlns:w16cex"): # type: ignore + root.setAttribute( # type: ignore + "xmlns:w16cex", + "http://schemas.microsoft.com/office/word/2018/wordml/cex", + ) + + def _ensure_w14_namespace(self): + """Ensure w14 namespace is declared on the root element.""" + root = self.dom.documentElement + if not root.hasAttribute("xmlns:w14"): # type: ignore + root.setAttribute( # type: ignore + "xmlns:w14", + "http://schemas.microsoft.com/office/word/2010/wordml", + ) + + def _inject_attributes_to_nodes(self, nodes): + """Inject RSID, author, and date attributes into DOM nodes where applicable. + + Adds attributes to elements that support them: + - w:r: gets w:rsidR (or w:rsidDel if inside w:del) + - w:p: gets w:rsidR, w:rsidRDefault, w:rsidP, w14:paraId, w14:textId + - w:t: gets xml:space="preserve" if text has leading/trailing whitespace + - w:ins, w:del: get w:id, w:author, w:date, w16du:dateUtc + - w:comment: gets w:author, w:date, w:initials + - w16cex:commentExtensible: gets w16cex:dateUtc + + Args: + nodes: List of DOM nodes to process + """ + from datetime import datetime, timezone + + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + def is_inside_deletion(elem): + """Check if element is inside a w:del element.""" + parent = elem.parentNode + while parent: + if parent.nodeType == parent.ELEMENT_NODE and parent.tagName == "w:del": + return True + parent = parent.parentNode + return False + + def add_rsid_to_p(elem): + if not elem.hasAttribute("w:rsidR"): + elem.setAttribute("w:rsidR", self.rsid) + if not elem.hasAttribute("w:rsidRDefault"): + elem.setAttribute("w:rsidRDefault", self.rsid) + if not elem.hasAttribute("w:rsidP"): + elem.setAttribute("w:rsidP", self.rsid) + # Add w14:paraId and w14:textId if not present + if not elem.hasAttribute("w14:paraId"): + self._ensure_w14_namespace() + elem.setAttribute("w14:paraId", _generate_hex_id()) + if not elem.hasAttribute("w14:textId"): + self._ensure_w14_namespace() + elem.setAttribute("w14:textId", _generate_hex_id()) + + def add_rsid_to_r(elem): + # Use w:rsidDel for inside , otherwise w:rsidR + if is_inside_deletion(elem): + if not elem.hasAttribute("w:rsidDel"): + elem.setAttribute("w:rsidDel", self.rsid) + else: + if not elem.hasAttribute("w:rsidR"): + elem.setAttribute("w:rsidR", self.rsid) + + def add_tracked_change_attrs(elem): + # Auto-assign w:id if not present + if not elem.hasAttribute("w:id"): + elem.setAttribute("w:id", str(self._get_next_change_id())) + if not elem.hasAttribute("w:author"): + elem.setAttribute("w:author", self.author) + if not elem.hasAttribute("w:date"): + elem.setAttribute("w:date", timestamp) + # Add w16du:dateUtc for tracked changes (same as w:date since we generate UTC timestamps) + if elem.tagName in ("w:ins", "w:del") and not elem.hasAttribute( + "w16du:dateUtc" + ): + self._ensure_w16du_namespace() + elem.setAttribute("w16du:dateUtc", timestamp) + + def add_comment_attrs(elem): + if not elem.hasAttribute("w:author"): + elem.setAttribute("w:author", self.author) + if not elem.hasAttribute("w:date"): + elem.setAttribute("w:date", timestamp) + if not elem.hasAttribute("w:initials"): + elem.setAttribute("w:initials", self.initials) + + def add_comment_extensible_date(elem): + # Add w16cex:dateUtc for comment extensible elements + if not elem.hasAttribute("w16cex:dateUtc"): + self._ensure_w16cex_namespace() + elem.setAttribute("w16cex:dateUtc", timestamp) + + def add_xml_space_to_t(elem): + # Add xml:space="preserve" to w:t if text has leading/trailing whitespace + if ( + elem.firstChild + and elem.firstChild.nodeType == elem.firstChild.TEXT_NODE + ): + text = elem.firstChild.data + if text and (text[0].isspace() or text[-1].isspace()): + if not elem.hasAttribute("xml:space"): + elem.setAttribute("xml:space", "preserve") + + for node in nodes: + if node.nodeType != node.ELEMENT_NODE: + continue + + # Handle the node itself + if node.tagName == "w:p": + add_rsid_to_p(node) + elif node.tagName == "w:r": + add_rsid_to_r(node) + elif node.tagName == "w:t": + add_xml_space_to_t(node) + elif node.tagName in ("w:ins", "w:del"): + add_tracked_change_attrs(node) + elif node.tagName == "w:comment": + add_comment_attrs(node) + elif node.tagName == "w16cex:commentExtensible": + add_comment_extensible_date(node) + + # Process descendants (getElementsByTagName doesn't return the element itself) + for elem in node.getElementsByTagName("w:p"): + add_rsid_to_p(elem) + for elem in node.getElementsByTagName("w:r"): + add_rsid_to_r(elem) + for elem in node.getElementsByTagName("w:t"): + add_xml_space_to_t(elem) + for tag in ("w:ins", "w:del"): + for elem in node.getElementsByTagName(tag): + add_tracked_change_attrs(elem) + for elem in node.getElementsByTagName("w:comment"): + add_comment_attrs(elem) + for elem in node.getElementsByTagName("w16cex:commentExtensible"): + add_comment_extensible_date(elem) + + def replace_node(self, elem, new_content): + """Replace node with automatic attribute injection.""" + nodes = super().replace_node(elem, new_content) + self._inject_attributes_to_nodes(nodes) + return nodes + + def insert_after(self, elem, xml_content): + """Insert after with automatic attribute injection.""" + nodes = super().insert_after(elem, xml_content) + self._inject_attributes_to_nodes(nodes) + return nodes + + def insert_before(self, elem, xml_content): + """Insert before with automatic attribute injection.""" + nodes = super().insert_before(elem, xml_content) + self._inject_attributes_to_nodes(nodes) + return nodes + + def append_to(self, elem, xml_content): + """Append to with automatic attribute injection.""" + nodes = super().append_to(elem, xml_content) + self._inject_attributes_to_nodes(nodes) + return nodes + + def revert_insertion(self, elem): + """Reject an insertion by wrapping its content in a deletion. + + Wraps all runs inside w:ins in w:del, converting w:t to w:delText. + Can process a single w:ins element or a container element with multiple w:ins. + + Args: + elem: Element to process (w:ins, w:p, w:body, etc.) + + Returns: + list: List containing the processed element(s) + + Raises: + ValueError: If the element contains no w:ins elements + + Example: + # Reject a single insertion + ins = doc["word/document.xml"].get_node(tag="w:ins", attrs={"w:id": "5"}) + doc["word/document.xml"].revert_insertion(ins) + + # Reject all insertions in a paragraph + para = doc["word/document.xml"].get_node(tag="w:p", line_number=42) + doc["word/document.xml"].revert_insertion(para) + """ + # Collect insertions + ins_elements = [] + if elem.tagName == "w:ins": + ins_elements.append(elem) + else: + ins_elements.extend(elem.getElementsByTagName("w:ins")) + + # Validate that there are insertions to reject + if not ins_elements: + raise ValueError( + f"revert_insertion requires w:ins elements. " + f"The provided element <{elem.tagName}> contains no insertions. " + ) + + # Process all insertions - wrap all children in w:del + for ins_elem in ins_elements: + runs = list(ins_elem.getElementsByTagName("w:r")) + if not runs: + continue + + # Create deletion wrapper + del_wrapper = self.dom.createElement("w:del") + + # Process each run + for run in runs: + # Convert w:t → w:delText and w:rsidR → w:rsidDel + if run.hasAttribute("w:rsidR"): + run.setAttribute("w:rsidDel", run.getAttribute("w:rsidR")) + run.removeAttribute("w:rsidR") + elif not run.hasAttribute("w:rsidDel"): + run.setAttribute("w:rsidDel", self.rsid) + + for t_elem in list(run.getElementsByTagName("w:t")): + del_text = self.dom.createElement("w:delText") + # Copy ALL child nodes (not just firstChild) to handle entities + while t_elem.firstChild: + del_text.appendChild(t_elem.firstChild) + for i in range(t_elem.attributes.length): + attr = t_elem.attributes.item(i) + del_text.setAttribute(attr.name, attr.value) + t_elem.parentNode.replaceChild(del_text, t_elem) + + # Move all children from ins to del wrapper + while ins_elem.firstChild: + del_wrapper.appendChild(ins_elem.firstChild) + + # Add del wrapper back to ins + ins_elem.appendChild(del_wrapper) + + # Inject attributes to the deletion wrapper + self._inject_attributes_to_nodes([del_wrapper]) + + return [elem] + + def revert_deletion(self, elem): + """Reject a deletion by re-inserting the deleted content. + + Creates w:ins elements after each w:del, copying deleted content and + converting w:delText back to w:t. + Can process a single w:del element or a container element with multiple w:del. + + Args: + elem: Element to process (w:del, w:p, w:body, etc.) + + Returns: + list: If elem is w:del, returns [elem, new_ins]. Otherwise returns [elem]. + + Raises: + ValueError: If the element contains no w:del elements + + Example: + # Reject a single deletion - returns [w:del, w:ins] + del_elem = doc["word/document.xml"].get_node(tag="w:del", attrs={"w:id": "3"}) + nodes = doc["word/document.xml"].revert_deletion(del_elem) + + # Reject all deletions in a paragraph - returns [para] + para = doc["word/document.xml"].get_node(tag="w:p", line_number=42) + nodes = doc["word/document.xml"].revert_deletion(para) + """ + # Collect deletions FIRST - before we modify the DOM + del_elements = [] + is_single_del = elem.tagName == "w:del" + + if is_single_del: + del_elements.append(elem) + else: + del_elements.extend(elem.getElementsByTagName("w:del")) + + # Validate that there are deletions to reject + if not del_elements: + raise ValueError( + f"revert_deletion requires w:del elements. " + f"The provided element <{elem.tagName}> contains no deletions. " + ) + + # Track created insertion (only relevant if elem is a single w:del) + created_insertion = None + + # Process all deletions - create insertions that copy the deleted content + for del_elem in del_elements: + # Clone the deleted runs and convert them to insertions + runs = list(del_elem.getElementsByTagName("w:r")) + if not runs: + continue + + # Create insertion wrapper + ins_elem = self.dom.createElement("w:ins") + + for run in runs: + # Clone the run + new_run = run.cloneNode(True) + + # Convert w:delText → w:t + for del_text in list(new_run.getElementsByTagName("w:delText")): + t_elem = self.dom.createElement("w:t") + # Copy ALL child nodes (not just firstChild) to handle entities + while del_text.firstChild: + t_elem.appendChild(del_text.firstChild) + for i in range(del_text.attributes.length): + attr = del_text.attributes.item(i) + t_elem.setAttribute(attr.name, attr.value) + del_text.parentNode.replaceChild(t_elem, del_text) + + # Update run attributes: w:rsidDel → w:rsidR + if new_run.hasAttribute("w:rsidDel"): + new_run.setAttribute("w:rsidR", new_run.getAttribute("w:rsidDel")) + new_run.removeAttribute("w:rsidDel") + elif not new_run.hasAttribute("w:rsidR"): + new_run.setAttribute("w:rsidR", self.rsid) + + ins_elem.appendChild(new_run) + + # Insert the new insertion after the deletion + nodes = self.insert_after(del_elem, ins_elem.toxml()) + + # If processing a single w:del, track the created insertion + if is_single_del and nodes: + created_insertion = nodes[0] + + # Return based on input type + if is_single_del and created_insertion: + return [elem, created_insertion] + else: + return [elem] + + @staticmethod + def suggest_paragraph(xml_content: str) -> str: + """Transform paragraph XML to add tracked change wrapping for insertion. + + Wraps runs in and adds to w:rPr in w:pPr for numbered lists. + + Args: + xml_content: XML string containing a element + + Returns: + str: Transformed XML with tracked change wrapping + """ + wrapper = f'{xml_content}' + doc = minidom.parseString(wrapper) + para = doc.getElementsByTagName("w:p")[0] + + # Ensure w:pPr exists + pPr_list = para.getElementsByTagName("w:pPr") + if not pPr_list: + pPr = doc.createElement("w:pPr") + para.insertBefore( + pPr, para.firstChild + ) if para.firstChild else para.appendChild(pPr) + else: + pPr = pPr_list[0] + + # Ensure w:rPr exists in w:pPr + rPr_list = pPr.getElementsByTagName("w:rPr") + if not rPr_list: + rPr = doc.createElement("w:rPr") + pPr.appendChild(rPr) + else: + rPr = rPr_list[0] + + # Add to w:rPr + ins_marker = doc.createElement("w:ins") + rPr.insertBefore( + ins_marker, rPr.firstChild + ) if rPr.firstChild else rPr.appendChild(ins_marker) + + # Wrap all non-pPr children in + ins_wrapper = doc.createElement("w:ins") + for child in [c for c in para.childNodes if c.nodeName != "w:pPr"]: + para.removeChild(child) + ins_wrapper.appendChild(child) + para.appendChild(ins_wrapper) + + return para.toxml() + + def suggest_deletion(self, elem): + """Mark a w:r or w:p element as deleted with tracked changes (in-place DOM manipulation). + + For w:r: wraps in , converts to , preserves w:rPr + For w:p (regular): wraps content in , converts to + For w:p (numbered list): adds to w:rPr in w:pPr, wraps content in + + Args: + elem: A w:r or w:p DOM element without existing tracked changes + + Returns: + Element: The modified element + + Raises: + ValueError: If element has existing tracked changes or invalid structure + """ + if elem.nodeName == "w:r": + # Check for existing w:delText + if elem.getElementsByTagName("w:delText"): + raise ValueError("w:r element already contains w:delText") + + # Convert w:t → w:delText + for t_elem in list(elem.getElementsByTagName("w:t")): + del_text = self.dom.createElement("w:delText") + # Copy ALL child nodes (not just firstChild) to handle entities + while t_elem.firstChild: + del_text.appendChild(t_elem.firstChild) + # Preserve attributes like xml:space + for i in range(t_elem.attributes.length): + attr = t_elem.attributes.item(i) + del_text.setAttribute(attr.name, attr.value) + t_elem.parentNode.replaceChild(del_text, t_elem) + + # Update run attributes: w:rsidR → w:rsidDel + if elem.hasAttribute("w:rsidR"): + elem.setAttribute("w:rsidDel", elem.getAttribute("w:rsidR")) + elem.removeAttribute("w:rsidR") + elif not elem.hasAttribute("w:rsidDel"): + elem.setAttribute("w:rsidDel", self.rsid) + + # Wrap in w:del + del_wrapper = self.dom.createElement("w:del") + parent = elem.parentNode + parent.insertBefore(del_wrapper, elem) + parent.removeChild(elem) + del_wrapper.appendChild(elem) + + # Inject attributes to the deletion wrapper + self._inject_attributes_to_nodes([del_wrapper]) + + return del_wrapper + + elif elem.nodeName == "w:p": + # Check for existing tracked changes + if elem.getElementsByTagName("w:ins") or elem.getElementsByTagName("w:del"): + raise ValueError("w:p element already contains tracked changes") + + # Check if it's a numbered list item + pPr_list = elem.getElementsByTagName("w:pPr") + is_numbered = pPr_list and pPr_list[0].getElementsByTagName("w:numPr") + + if is_numbered: + # Add to w:rPr in w:pPr + pPr = pPr_list[0] + rPr_list = pPr.getElementsByTagName("w:rPr") + + if not rPr_list: + rPr = self.dom.createElement("w:rPr") + pPr.appendChild(rPr) + else: + rPr = rPr_list[0] + + # Add marker + del_marker = self.dom.createElement("w:del") + rPr.insertBefore( + del_marker, rPr.firstChild + ) if rPr.firstChild else rPr.appendChild(del_marker) + + # Convert w:t → w:delText in all runs + for t_elem in list(elem.getElementsByTagName("w:t")): + del_text = self.dom.createElement("w:delText") + # Copy ALL child nodes (not just firstChild) to handle entities + while t_elem.firstChild: + del_text.appendChild(t_elem.firstChild) + # Preserve attributes like xml:space + for i in range(t_elem.attributes.length): + attr = t_elem.attributes.item(i) + del_text.setAttribute(attr.name, attr.value) + t_elem.parentNode.replaceChild(del_text, t_elem) + + # Update run attributes: w:rsidR → w:rsidDel + for run in elem.getElementsByTagName("w:r"): + if run.hasAttribute("w:rsidR"): + run.setAttribute("w:rsidDel", run.getAttribute("w:rsidR")) + run.removeAttribute("w:rsidR") + elif not run.hasAttribute("w:rsidDel"): + run.setAttribute("w:rsidDel", self.rsid) + + # Wrap all non-pPr children in + del_wrapper = self.dom.createElement("w:del") + for child in [c for c in elem.childNodes if c.nodeName != "w:pPr"]: + elem.removeChild(child) + del_wrapper.appendChild(child) + elem.appendChild(del_wrapper) + + # Inject attributes to the deletion wrapper + self._inject_attributes_to_nodes([del_wrapper]) + + return elem + + else: + raise ValueError(f"Element must be w:r or w:p, got {elem.nodeName}") + + +def _generate_hex_id() -> str: + """Generate random 8-character hex ID for para/durable IDs. + + Values are constrained to be less than 0x7FFFFFFF per OOXML spec: + - paraId must be < 0x80000000 + - durableId must be < 0x7FFFFFFF + We use the stricter constraint (0x7FFFFFFF) for both. + """ + return f"{random.randint(1, 0x7FFFFFFE):08X}" + + +def _generate_rsid() -> str: + """Generate random 8-character hex RSID.""" + return "".join(random.choices("0123456789ABCDEF", k=8)) + + +class Document: + """Manages comments in unpacked Word documents.""" + + def __init__( + self, + unpacked_dir, + rsid=None, + track_revisions=False, + author="Claude", + initials="C", + ): + """ + Initialize with path to unpacked Word document directory. + Automatically sets up comment infrastructure (people.xml, RSIDs). + + Args: + unpacked_dir: Path to unpacked DOCX directory (must contain word/ subdirectory) + rsid: Optional RSID to use for all comment elements. If not provided, one will be generated. + track_revisions: If True, enables track revisions in settings.xml (default: False) + author: Default author name for comments (default: "Claude") + initials: Default author initials for comments (default: "C") + """ + self.original_path = Path(unpacked_dir) + + if not self.original_path.exists() or not self.original_path.is_dir(): + raise ValueError(f"Directory not found: {unpacked_dir}") + + # Create temporary directory with subdirectories for unpacked content and baseline + self.temp_dir = tempfile.mkdtemp(prefix="docx_") + self.unpacked_path = Path(self.temp_dir) / "unpacked" + shutil.copytree(self.original_path, self.unpacked_path) + + # Pack original directory into temporary .docx for validation baseline (outside unpacked dir) + self.original_docx = Path(self.temp_dir) / "original.docx" + pack_document(self.original_path, self.original_docx, validate=False) + + self.word_path = self.unpacked_path / "word" + + # Generate RSID if not provided + self.rsid = rsid if rsid else _generate_rsid() + print(f"Using RSID: {self.rsid}") + + # Set default author and initials + self.author = author + self.initials = initials + + # Cache for lazy-loaded editors + self._editors = {} + + # Comment file paths + self.comments_path = self.word_path / "comments.xml" + self.comments_extended_path = self.word_path / "commentsExtended.xml" + self.comments_ids_path = self.word_path / "commentsIds.xml" + self.comments_extensible_path = self.word_path / "commentsExtensible.xml" + + # Load existing comments and determine next ID (before setup modifies files) + self.existing_comments = self._load_existing_comments() + self.next_comment_id = self._get_next_comment_id() + + # Convenient access to document.xml editor (semi-private) + self._document = self["word/document.xml"] + + # Setup tracked changes infrastructure + self._setup_tracking(track_revisions=track_revisions) + + # Add author to people.xml + self._add_author_to_people(author) + + def __getitem__(self, xml_path: str) -> DocxXMLEditor: + """ + Get or create a DocxXMLEditor for the specified XML file. + + Enables lazy-loaded editors with bracket notation: + node = doc["word/document.xml"].get_node(tag="w:p", line_number=42) + + Args: + xml_path: Relative path to XML file (e.g., "word/document.xml", "word/comments.xml") + + Returns: + DocxXMLEditor instance for the specified file + + Raises: + ValueError: If the file does not exist + + Example: + # Get node from document.xml + node = doc["word/document.xml"].get_node(tag="w:del", attrs={"w:id": "1"}) + + # Get node from comments.xml + comment = doc["word/comments.xml"].get_node(tag="w:comment", attrs={"w:id": "0"}) + """ + if xml_path not in self._editors: + file_path = self.unpacked_path / xml_path + if not file_path.exists(): + raise ValueError(f"XML file not found: {xml_path}") + # Use DocxXMLEditor with RSID, author, and initials for all editors + self._editors[xml_path] = DocxXMLEditor( + file_path, rsid=self.rsid, author=self.author, initials=self.initials + ) + return self._editors[xml_path] + + def add_comment(self, start, end, text: str) -> int: + """ + Add a comment spanning from one element to another. + + Args: + start: DOM element for the starting point + end: DOM element for the ending point + text: Comment content + + Returns: + The comment ID that was created + + Example: + start_node = cm.get_document_node(tag="w:del", id="1") + end_node = cm.get_document_node(tag="w:ins", id="2") + cm.add_comment(start=start_node, end=end_node, text="Explanation") + """ + comment_id = self.next_comment_id + para_id = _generate_hex_id() + durable_id = _generate_hex_id() + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + # Add comment ranges to document.xml immediately + self._document.insert_before(start, self._comment_range_start_xml(comment_id)) + + # If end node is a paragraph, append comment markup inside it + # Otherwise insert after it (for run-level anchors) + if end.tagName == "w:p": + self._document.append_to(end, self._comment_range_end_xml(comment_id)) + else: + self._document.insert_after(end, self._comment_range_end_xml(comment_id)) + + # Add to comments.xml immediately + self._add_to_comments_xml( + comment_id, para_id, text, self.author, self.initials, timestamp + ) + + # Add to commentsExtended.xml immediately + self._add_to_comments_extended_xml(para_id, parent_para_id=None) + + # Add to commentsIds.xml immediately + self._add_to_comments_ids_xml(para_id, durable_id) + + # Add to commentsExtensible.xml immediately + self._add_to_comments_extensible_xml(durable_id) + + # Update existing_comments so replies work + self.existing_comments[comment_id] = {"para_id": para_id} + + self.next_comment_id += 1 + return comment_id + + def reply_to_comment( + self, + parent_comment_id: int, + text: str, + ) -> int: + """ + Add a reply to an existing comment. + + Args: + parent_comment_id: The w:id of the parent comment to reply to + text: Reply text + + Returns: + The comment ID that was created for the reply + + Example: + cm.reply_to_comment(parent_comment_id=0, text="I agree with this change") + """ + if parent_comment_id not in self.existing_comments: + raise ValueError(f"Parent comment with id={parent_comment_id} not found") + + parent_info = self.existing_comments[parent_comment_id] + comment_id = self.next_comment_id + para_id = _generate_hex_id() + durable_id = _generate_hex_id() + timestamp = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + # Add comment ranges to document.xml immediately + parent_start_elem = self._document.get_node( + tag="w:commentRangeStart", attrs={"w:id": str(parent_comment_id)} + ) + parent_ref_elem = self._document.get_node( + tag="w:commentReference", attrs={"w:id": str(parent_comment_id)} + ) + + self._document.insert_after( + parent_start_elem, self._comment_range_start_xml(comment_id) + ) + parent_ref_run = parent_ref_elem.parentNode + self._document.insert_after( + parent_ref_run, f'' + ) + self._document.insert_after( + parent_ref_run, self._comment_ref_run_xml(comment_id) + ) + + # Add to comments.xml immediately + self._add_to_comments_xml( + comment_id, para_id, text, self.author, self.initials, timestamp + ) + + # Add to commentsExtended.xml immediately (with parent) + self._add_to_comments_extended_xml( + para_id, parent_para_id=parent_info["para_id"] + ) + + # Add to commentsIds.xml immediately + self._add_to_comments_ids_xml(para_id, durable_id) + + # Add to commentsExtensible.xml immediately + self._add_to_comments_extensible_xml(durable_id) + + # Update existing_comments so replies work + self.existing_comments[comment_id] = {"para_id": para_id} + + self.next_comment_id += 1 + return comment_id + + def __del__(self): + """Clean up temporary directory on deletion.""" + if hasattr(self, "temp_dir") and Path(self.temp_dir).exists(): + shutil.rmtree(self.temp_dir) + + def validate(self) -> None: + """ + Validate the document against XSD schema and redlining rules. + + Raises: + ValueError: If validation fails. + """ + # Create validators with current state + schema_validator = DOCXSchemaValidator( + self.unpacked_path, self.original_docx, verbose=False + ) + redlining_validator = RedliningValidator( + self.unpacked_path, self.original_docx, verbose=False + ) + + # Run validations + if not schema_validator.validate(): + raise ValueError("Schema validation failed") + if not redlining_validator.validate(): + raise ValueError("Redlining validation failed") + + def save(self, destination=None, validate=True) -> None: + """ + Save all modified XML files to disk and copy to destination directory. + + This persists all changes made via add_comment() and reply_to_comment(). + + Args: + destination: Optional path to save to. If None, saves back to original directory. + validate: If True, validates document before saving (default: True). + """ + # Only ensure comment relationships and content types if comment files exist + if self.comments_path.exists(): + self._ensure_comment_relationships() + self._ensure_comment_content_types() + + # Save all modified XML files in temp directory + for editor in self._editors.values(): + editor.save() + + # Validate by default + if validate: + self.validate() + + # Copy contents from temp directory to destination (or original directory) + target_path = Path(destination) if destination else self.original_path + shutil.copytree(self.unpacked_path, target_path, dirs_exist_ok=True) + + # ==================== Private: Initialization ==================== + + def _get_next_comment_id(self): + """Get the next available comment ID.""" + if not self.comments_path.exists(): + return 0 + + editor = self["word/comments.xml"] + max_id = -1 + for comment_elem in editor.dom.getElementsByTagName("w:comment"): + comment_id = comment_elem.getAttribute("w:id") + if comment_id: + try: + max_id = max(max_id, int(comment_id)) + except ValueError: + pass + return max_id + 1 + + def _load_existing_comments(self): + """Load existing comments from files to enable replies.""" + if not self.comments_path.exists(): + return {} + + editor = self["word/comments.xml"] + existing = {} + + for comment_elem in editor.dom.getElementsByTagName("w:comment"): + comment_id = comment_elem.getAttribute("w:id") + if not comment_id: + continue + + # Find para_id from the w:p element within the comment + para_id = None + for p_elem in comment_elem.getElementsByTagName("w:p"): + para_id = p_elem.getAttribute("w14:paraId") + if para_id: + break + + if not para_id: + continue + + existing[int(comment_id)] = {"para_id": para_id} + + return existing + + # ==================== Private: Setup Methods ==================== + + def _setup_tracking(self, track_revisions=False): + """Set up comment infrastructure in unpacked directory. + + Args: + track_revisions: If True, enables track revisions in settings.xml + """ + # Create or update word/people.xml + people_file = self.word_path / "people.xml" + self._update_people_xml(people_file) + + # Update XML files + self._add_content_type_for_people(self.unpacked_path / "[Content_Types].xml") + self._add_relationship_for_people( + self.word_path / "_rels" / "document.xml.rels" + ) + + # Always add RSID to settings.xml, optionally enable trackRevisions + self._update_settings( + self.word_path / "settings.xml", track_revisions=track_revisions + ) + + def _update_people_xml(self, path): + """Create people.xml if it doesn't exist.""" + if not path.exists(): + # Copy from template + shutil.copy(TEMPLATE_DIR / "people.xml", path) + + def _add_content_type_for_people(self, path): + """Add people.xml content type to [Content_Types].xml if not already present.""" + editor = self["[Content_Types].xml"] + + if self._has_override(editor, "/word/people.xml"): + return + + # Add Override element + root = editor.dom.documentElement + override_xml = '' + editor.append_to(root, override_xml) + + def _add_relationship_for_people(self, path): + """Add people.xml relationship to document.xml.rels if not already present.""" + editor = self["word/_rels/document.xml.rels"] + + if self._has_relationship(editor, "people.xml"): + return + + root = editor.dom.documentElement + root_tag = root.tagName # type: ignore + prefix = root_tag.split(":")[0] + ":" if ":" in root_tag else "" + next_rid = editor.get_next_rid() + + # Create the relationship entry + rel_xml = f'<{prefix}Relationship Id="{next_rid}" Type="http://schemas.microsoft.com/office/2011/relationships/people" Target="people.xml"/>' + editor.append_to(root, rel_xml) + + def _update_settings(self, path, track_revisions=False): + """Add RSID and optionally enable track revisions in settings.xml. + + Args: + path: Path to settings.xml + track_revisions: If True, adds trackRevisions element + + Places elements per OOXML schema order: + - trackRevisions: early (before defaultTabStop) + - rsids: late (after compat) + """ + editor = self["word/settings.xml"] + root = editor.get_node(tag="w:settings") + prefix = root.tagName.split(":")[0] if ":" in root.tagName else "w" + + # Conditionally add trackRevisions if requested + if track_revisions: + track_revisions_exists = any( + elem.tagName == f"{prefix}:trackRevisions" + for elem in editor.dom.getElementsByTagName(f"{prefix}:trackRevisions") + ) + + if not track_revisions_exists: + track_rev_xml = f"<{prefix}:trackRevisions/>" + # Try to insert before documentProtection, defaultTabStop, or at start + inserted = False + for tag in [f"{prefix}:documentProtection", f"{prefix}:defaultTabStop"]: + elements = editor.dom.getElementsByTagName(tag) + if elements: + editor.insert_before(elements[0], track_rev_xml) + inserted = True + break + if not inserted: + # Insert as first child of settings + if root.firstChild: + editor.insert_before(root.firstChild, track_rev_xml) + else: + editor.append_to(root, track_rev_xml) + + # Always check if rsids section exists + rsids_elements = editor.dom.getElementsByTagName(f"{prefix}:rsids") + + if not rsids_elements: + # Add new rsids section + rsids_xml = f'''<{prefix}:rsids> + <{prefix}:rsidRoot {prefix}:val="{self.rsid}"/> + <{prefix}:rsid {prefix}:val="{self.rsid}"/> +''' + + # Try to insert after compat, before clrSchemeMapping, or before closing tag + inserted = False + compat_elements = editor.dom.getElementsByTagName(f"{prefix}:compat") + if compat_elements: + editor.insert_after(compat_elements[0], rsids_xml) + inserted = True + + if not inserted: + clr_elements = editor.dom.getElementsByTagName( + f"{prefix}:clrSchemeMapping" + ) + if clr_elements: + editor.insert_before(clr_elements[0], rsids_xml) + inserted = True + + if not inserted: + editor.append_to(root, rsids_xml) + else: + # Check if this rsid already exists + rsids_elem = rsids_elements[0] + rsid_exists = any( + elem.getAttribute(f"{prefix}:val") == self.rsid + for elem in rsids_elem.getElementsByTagName(f"{prefix}:rsid") + ) + + if not rsid_exists: + rsid_xml = f'<{prefix}:rsid {prefix}:val="{self.rsid}"/>' + editor.append_to(rsids_elem, rsid_xml) + + # ==================== Private: XML File Creation ==================== + + def _add_to_comments_xml( + self, comment_id, para_id, text, author, initials, timestamp + ): + """Add a single comment to comments.xml.""" + if not self.comments_path.exists(): + shutil.copy(TEMPLATE_DIR / "comments.xml", self.comments_path) + + editor = self["word/comments.xml"] + root = editor.get_node(tag="w:comments") + + escaped_text = ( + text.replace("&", "&").replace("<", "<").replace(">", ">") + ) + # Note: w:rsidR, w:rsidRDefault, w:rsidP on w:p, w:rsidR on w:r, + # and w:author, w:date, w:initials on w:comment are automatically added by DocxXMLEditor + comment_xml = f''' + + + {escaped_text} + +''' + editor.append_to(root, comment_xml) + + def _add_to_comments_extended_xml(self, para_id, parent_para_id): + """Add a single comment to commentsExtended.xml.""" + if not self.comments_extended_path.exists(): + shutil.copy( + TEMPLATE_DIR / "commentsExtended.xml", self.comments_extended_path + ) + + editor = self["word/commentsExtended.xml"] + root = editor.get_node(tag="w15:commentsEx") + + if parent_para_id: + xml = f'' + else: + xml = f'' + editor.append_to(root, xml) + + def _add_to_comments_ids_xml(self, para_id, durable_id): + """Add a single comment to commentsIds.xml.""" + if not self.comments_ids_path.exists(): + shutil.copy(TEMPLATE_DIR / "commentsIds.xml", self.comments_ids_path) + + editor = self["word/commentsIds.xml"] + root = editor.get_node(tag="w16cid:commentsIds") + + xml = f'' + editor.append_to(root, xml) + + def _add_to_comments_extensible_xml(self, durable_id): + """Add a single comment to commentsExtensible.xml.""" + if not self.comments_extensible_path.exists(): + shutil.copy( + TEMPLATE_DIR / "commentsExtensible.xml", self.comments_extensible_path + ) + + editor = self["word/commentsExtensible.xml"] + root = editor.get_node(tag="w16cex:commentsExtensible") + + xml = f'' + editor.append_to(root, xml) + + # ==================== Private: XML Fragments ==================== + + def _comment_range_start_xml(self, comment_id): + """Generate XML for comment range start.""" + return f'' + + def _comment_range_end_xml(self, comment_id): + """Generate XML for comment range end with reference run. + + Note: w:rsidR is automatically added by DocxXMLEditor. + """ + return f''' + + + +''' + + def _comment_ref_run_xml(self, comment_id): + """Generate XML for comment reference run. + + Note: w:rsidR is automatically added by DocxXMLEditor. + """ + return f''' + + +''' + + # ==================== Private: Metadata Updates ==================== + + def _has_relationship(self, editor, target): + """Check if a relationship with given target exists.""" + for rel_elem in editor.dom.getElementsByTagName("Relationship"): + if rel_elem.getAttribute("Target") == target: + return True + return False + + def _has_override(self, editor, part_name): + """Check if an override with given part name exists.""" + for override_elem in editor.dom.getElementsByTagName("Override"): + if override_elem.getAttribute("PartName") == part_name: + return True + return False + + def _has_author(self, editor, author): + """Check if an author already exists in people.xml.""" + for person_elem in editor.dom.getElementsByTagName("w15:person"): + if person_elem.getAttribute("w15:author") == author: + return True + return False + + def _add_author_to_people(self, author): + """Add author to people.xml (called during initialization).""" + people_path = self.word_path / "people.xml" + + # people.xml should already exist from _setup_tracking + if not people_path.exists(): + raise ValueError("people.xml should exist after _setup_tracking") + + editor = self["word/people.xml"] + root = editor.get_node(tag="w15:people") + + # Check if author already exists + if self._has_author(editor, author): + return + + # Add author with proper XML escaping to prevent injection + escaped_author = html.escape(author, quote=True) + person_xml = f''' + +''' + editor.append_to(root, person_xml) + + def _ensure_comment_relationships(self): + """Ensure word/_rels/document.xml.rels has comment relationships.""" + editor = self["word/_rels/document.xml.rels"] + + if self._has_relationship(editor, "comments.xml"): + return + + root = editor.dom.documentElement + root_tag = root.tagName # type: ignore + prefix = root_tag.split(":")[0] + ":" if ":" in root_tag else "" + next_rid_num = int(editor.get_next_rid()[3:]) + + # Add relationship elements + rels = [ + ( + next_rid_num, + "http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments", + "comments.xml", + ), + ( + next_rid_num + 1, + "http://schemas.microsoft.com/office/2011/relationships/commentsExtended", + "commentsExtended.xml", + ), + ( + next_rid_num + 2, + "http://schemas.microsoft.com/office/2016/09/relationships/commentsIds", + "commentsIds.xml", + ), + ( + next_rid_num + 3, + "http://schemas.microsoft.com/office/2018/08/relationships/commentsExtensible", + "commentsExtensible.xml", + ), + ] + + for rel_id, rel_type, target in rels: + rel_xml = f'<{prefix}Relationship Id="rId{rel_id}" Type="{rel_type}" Target="{target}"/>' + editor.append_to(root, rel_xml) + + def _ensure_comment_content_types(self): + """Ensure [Content_Types].xml has comment content types.""" + editor = self["[Content_Types].xml"] + + if self._has_override(editor, "/word/comments.xml"): + return + + root = editor.dom.documentElement + + # Add Override elements + overrides = [ + ( + "/word/comments.xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml", + ), + ( + "/word/commentsExtended.xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtended+xml", + ), + ( + "/word/commentsIds.xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsIds+xml", + ), + ( + "/word/commentsExtensible.xml", + "application/vnd.openxmlformats-officedocument.wordprocessingml.commentsExtensible+xml", + ), + ] + + for part_name, content_type in overrides: + override_xml = ( + f'' + ) + editor.append_to(root, override_xml) diff --git a/skills/document-skills/docx/scripts/templates/comments.xml b/skills/document-skills/docx/scripts/templates/comments.xml new file mode 100644 index 0000000..b5dace0 --- /dev/null +++ b/skills/document-skills/docx/scripts/templates/comments.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/skills/document-skills/docx/scripts/templates/commentsExtended.xml b/skills/document-skills/docx/scripts/templates/commentsExtended.xml new file mode 100644 index 0000000..b4cf23e --- /dev/null +++ b/skills/document-skills/docx/scripts/templates/commentsExtended.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/skills/document-skills/docx/scripts/templates/commentsExtensible.xml b/skills/document-skills/docx/scripts/templates/commentsExtensible.xml new file mode 100644 index 0000000..e32a05e --- /dev/null +++ b/skills/document-skills/docx/scripts/templates/commentsExtensible.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/skills/document-skills/docx/scripts/templates/commentsIds.xml b/skills/document-skills/docx/scripts/templates/commentsIds.xml new file mode 100644 index 0000000..d04bc8e --- /dev/null +++ b/skills/document-skills/docx/scripts/templates/commentsIds.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/skills/document-skills/docx/scripts/templates/people.xml b/skills/document-skills/docx/scripts/templates/people.xml new file mode 100644 index 0000000..a839caf --- /dev/null +++ b/skills/document-skills/docx/scripts/templates/people.xml @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/skills/document-skills/docx/scripts/utilities.py b/skills/document-skills/docx/scripts/utilities.py new file mode 100755 index 0000000..d92dae6 --- /dev/null +++ b/skills/document-skills/docx/scripts/utilities.py @@ -0,0 +1,374 @@ +#!/usr/bin/env python3 +""" +Utilities for editing OOXML documents. + +This module provides XMLEditor, a tool for manipulating XML files with support for +line-number-based node finding and DOM manipulation. Each element is automatically +annotated with its original line and column position during parsing. + +Example usage: + editor = XMLEditor("document.xml") + + # Find node by line number or range + elem = editor.get_node(tag="w:r", line_number=519) + elem = editor.get_node(tag="w:p", line_number=range(100, 200)) + + # Find node by text content + elem = editor.get_node(tag="w:p", contains="specific text") + + # Find node by attributes + elem = editor.get_node(tag="w:r", attrs={"w:id": "target"}) + + # Combine filters + elem = editor.get_node(tag="w:p", line_number=range(1, 50), contains="text") + + # Replace, insert, or manipulate + new_elem = editor.replace_node(elem, "new text") + editor.insert_after(new_elem, "more") + + # Save changes + editor.save() +""" + +import html +from pathlib import Path +from typing import Optional, Union + +import defusedxml.minidom +import defusedxml.sax + + +class XMLEditor: + """ + Editor for manipulating OOXML XML files with line-number-based node finding. + + This class parses XML files and tracks the original line and column position + of each element. This enables finding nodes by their line number in the original + file, which is useful when working with Read tool output. + + Attributes: + xml_path: Path to the XML file being edited + encoding: Detected encoding of the XML file ('ascii' or 'utf-8') + dom: Parsed DOM tree with parse_position attributes on elements + """ + + def __init__(self, xml_path): + """ + Initialize with path to XML file and parse with line number tracking. + + Args: + xml_path: Path to XML file to edit (str or Path) + + Raises: + ValueError: If the XML file does not exist + """ + self.xml_path = Path(xml_path) + if not self.xml_path.exists(): + raise ValueError(f"XML file not found: {xml_path}") + + with open(self.xml_path, "rb") as f: + header = f.read(200).decode("utf-8", errors="ignore") + self.encoding = "ascii" if 'encoding="ascii"' in header else "utf-8" + + parser = _create_line_tracking_parser() + self.dom = defusedxml.minidom.parse(str(self.xml_path), parser) + + def get_node( + self, + tag: str, + attrs: Optional[dict[str, str]] = None, + line_number: Optional[Union[int, range]] = None, + contains: Optional[str] = None, + ): + """ + Get a DOM element by tag and identifier. + + Finds an element by either its line number in the original file or by + matching attribute values. Exactly one match must be found. + + Args: + tag: The XML tag name (e.g., "w:del", "w:ins", "w:r") + attrs: Dictionary of attribute name-value pairs to match (e.g., {"w:id": "1"}) + line_number: Line number (int) or line range (range) in original XML file (1-indexed) + contains: Text string that must appear in any text node within the element. + Supports both entity notation (“) and Unicode characters (\u201c). + + Returns: + defusedxml.minidom.Element: The matching DOM element + + Raises: + ValueError: If node not found or multiple matches found + + Example: + elem = editor.get_node(tag="w:r", line_number=519) + elem = editor.get_node(tag="w:r", line_number=range(100, 200)) + elem = editor.get_node(tag="w:del", attrs={"w:id": "1"}) + elem = editor.get_node(tag="w:p", attrs={"w14:paraId": "12345678"}) + elem = editor.get_node(tag="w:commentRangeStart", attrs={"w:id": "0"}) + elem = editor.get_node(tag="w:p", contains="specific text") + elem = editor.get_node(tag="w:t", contains="“Agreement") # Entity notation + elem = editor.get_node(tag="w:t", contains="\u201cAgreement") # Unicode character + """ + matches = [] + for elem in self.dom.getElementsByTagName(tag): + # Check line_number filter + if line_number is not None: + parse_pos = getattr(elem, "parse_position", (None,)) + elem_line = parse_pos[0] + + # Handle both single line number and range + if isinstance(line_number, range): + if elem_line not in line_number: + continue + else: + if elem_line != line_number: + continue + + # Check attrs filter + if attrs is not None: + if not all( + elem.getAttribute(attr_name) == attr_value + for attr_name, attr_value in attrs.items() + ): + continue + + # Check contains filter + if contains is not None: + elem_text = self._get_element_text(elem) + # Normalize the search string: convert HTML entities to Unicode characters + # This allows searching for both "“Rowan" and ""Rowan" + normalized_contains = html.unescape(contains) + if normalized_contains not in elem_text: + continue + + # If all applicable filters passed, this is a match + matches.append(elem) + + if not matches: + # Build descriptive error message + filters = [] + if line_number is not None: + line_str = ( + f"lines {line_number.start}-{line_number.stop - 1}" + if isinstance(line_number, range) + else f"line {line_number}" + ) + filters.append(f"at {line_str}") + if attrs is not None: + filters.append(f"with attributes {attrs}") + if contains is not None: + filters.append(f"containing '{contains}'") + + filter_desc = " ".join(filters) if filters else "" + base_msg = f"Node not found: <{tag}> {filter_desc}".strip() + + # Add helpful hint based on filters used + if contains: + hint = "Text may be split across elements or use different wording." + elif line_number: + hint = "Line numbers may have changed if document was modified." + elif attrs: + hint = "Verify attribute values are correct." + else: + hint = "Try adding filters (attrs, line_number, or contains)." + + raise ValueError(f"{base_msg}. {hint}") + if len(matches) > 1: + raise ValueError( + f"Multiple nodes found: <{tag}>. " + f"Add more filters (attrs, line_number, or contains) to narrow the search." + ) + return matches[0] + + def _get_element_text(self, elem): + """ + Recursively extract all text content from an element. + + Skips text nodes that contain only whitespace (spaces, tabs, newlines), + which typically represent XML formatting rather than document content. + + Args: + elem: defusedxml.minidom.Element to extract text from + + Returns: + str: Concatenated text from all non-whitespace text nodes within the element + """ + text_parts = [] + for node in elem.childNodes: + if node.nodeType == node.TEXT_NODE: + # Skip whitespace-only text nodes (XML formatting) + if node.data.strip(): + text_parts.append(node.data) + elif node.nodeType == node.ELEMENT_NODE: + text_parts.append(self._get_element_text(node)) + return "".join(text_parts) + + def replace_node(self, elem, new_content): + """ + Replace a DOM element with new XML content. + + Args: + elem: defusedxml.minidom.Element to replace + new_content: String containing XML to replace the node with + + Returns: + List[defusedxml.minidom.Node]: All inserted nodes + + Example: + new_nodes = editor.replace_node(old_elem, "text") + """ + parent = elem.parentNode + nodes = self._parse_fragment(new_content) + for node in nodes: + parent.insertBefore(node, elem) + parent.removeChild(elem) + return nodes + + def insert_after(self, elem, xml_content): + """ + Insert XML content after a DOM element. + + Args: + elem: defusedxml.minidom.Element to insert after + xml_content: String containing XML to insert + + Returns: + List[defusedxml.minidom.Node]: All inserted nodes + + Example: + new_nodes = editor.insert_after(elem, "text") + """ + parent = elem.parentNode + next_sibling = elem.nextSibling + nodes = self._parse_fragment(xml_content) + for node in nodes: + if next_sibling: + parent.insertBefore(node, next_sibling) + else: + parent.appendChild(node) + return nodes + + def insert_before(self, elem, xml_content): + """ + Insert XML content before a DOM element. + + Args: + elem: defusedxml.minidom.Element to insert before + xml_content: String containing XML to insert + + Returns: + List[defusedxml.minidom.Node]: All inserted nodes + + Example: + new_nodes = editor.insert_before(elem, "text") + """ + parent = elem.parentNode + nodes = self._parse_fragment(xml_content) + for node in nodes: + parent.insertBefore(node, elem) + return nodes + + def append_to(self, elem, xml_content): + """ + Append XML content as a child of a DOM element. + + Args: + elem: defusedxml.minidom.Element to append to + xml_content: String containing XML to append + + Returns: + List[defusedxml.minidom.Node]: All inserted nodes + + Example: + new_nodes = editor.append_to(elem, "text") + """ + nodes = self._parse_fragment(xml_content) + for node in nodes: + elem.appendChild(node) + return nodes + + def get_next_rid(self): + """Get the next available rId for relationships files.""" + max_id = 0 + for rel_elem in self.dom.getElementsByTagName("Relationship"): + rel_id = rel_elem.getAttribute("Id") + if rel_id.startswith("rId"): + try: + max_id = max(max_id, int(rel_id[3:])) + except ValueError: + pass + return f"rId{max_id + 1}" + + def save(self): + """ + Save the edited XML back to the file. + + Serializes the DOM tree and writes it back to the original file path, + preserving the original encoding (ascii or utf-8). + """ + content = self.dom.toxml(encoding=self.encoding) + self.xml_path.write_bytes(content) + + def _parse_fragment(self, xml_content): + """ + Parse XML fragment and return list of imported nodes. + + Args: + xml_content: String containing XML fragment + + Returns: + List of defusedxml.minidom.Node objects imported into this document + + Raises: + AssertionError: If fragment contains no element nodes + """ + # Extract namespace declarations from the root document element + root_elem = self.dom.documentElement + namespaces = [] + if root_elem and root_elem.attributes: + for i in range(root_elem.attributes.length): + attr = root_elem.attributes.item(i) + if attr.name.startswith("xmlns"): # type: ignore + namespaces.append(f'{attr.name}="{attr.value}"') # type: ignore + + ns_decl = " ".join(namespaces) + wrapper = f"{xml_content}" + fragment_doc = defusedxml.minidom.parseString(wrapper) + nodes = [ + self.dom.importNode(child, deep=True) + for child in fragment_doc.documentElement.childNodes # type: ignore + ] + elements = [n for n in nodes if n.nodeType == n.ELEMENT_NODE] + assert elements, "Fragment must contain at least one element" + return nodes + + +def _create_line_tracking_parser(): + """ + Create a SAX parser that tracks line and column numbers for each element. + + Monkey patches the SAX content handler to store the current line and column + position from the underlying expat parser onto each element as a parse_position + attribute (line, column) tuple. + + Returns: + defusedxml.sax.xmlreader.XMLReader: Configured SAX parser + """ + + def set_content_handler(dom_handler): + def startElementNS(name, tagName, attrs): + orig_start_cb(name, tagName, attrs) + cur_elem = dom_handler.elementStack[-1] + cur_elem.parse_position = ( + parser._parser.CurrentLineNumber, # type: ignore + parser._parser.CurrentColumnNumber, # type: ignore + ) + + orig_start_cb = dom_handler.startElementNS + dom_handler.startElementNS = startElementNS + orig_set_content_handler(dom_handler) + + parser = defusedxml.sax.make_parser() + orig_set_content_handler = parser.setContentHandler + parser.setContentHandler = set_content_handler # type: ignore + return parser diff --git a/skills/document-skills/pdf/LICENSE.txt b/skills/document-skills/pdf/LICENSE.txt new file mode 100644 index 0000000..c55ab42 --- /dev/null +++ b/skills/document-skills/pdf/LICENSE.txt @@ -0,0 +1,30 @@ +© 2025 Anthropic, PBC. All rights reserved. + +LICENSE: Use of these materials (including all code, prompts, assets, files, +and other components of this Skill) is governed by your agreement with +Anthropic regarding use of Anthropic's services. If no separate agreement +exists, use is governed by Anthropic's Consumer Terms of Service or +Commercial Terms of Service, as applicable: +https://www.anthropic.com/legal/consumer-terms +https://www.anthropic.com/legal/commercial-terms +Your applicable agreement is referred to as the "Agreement." "Services" are +as defined in the Agreement. + +ADDITIONAL RESTRICTIONS: Notwithstanding anything in the Agreement to the +contrary, users may not: + +- Extract these materials from the Services or retain copies of these + materials outside the Services +- Reproduce or copy these materials, except for temporary copies created + automatically during authorized use of the Services +- Create derivative works based on these materials +- Distribute, sublicense, or transfer these materials to any third party +- Make, offer to sell, sell, or import any inventions embodied in these + materials +- Reverse engineer, decompile, or disassemble these materials + +The receipt, viewing, or possession of these materials does not convey or +imply any license or right beyond those expressly granted above. + +Anthropic retains all right, title, and interest in these materials, +including all copyrights, patents, and other intellectual property rights. diff --git a/skills/document-skills/pdf/SKILL.md b/skills/document-skills/pdf/SKILL.md new file mode 100644 index 0000000..f6a22dd --- /dev/null +++ b/skills/document-skills/pdf/SKILL.md @@ -0,0 +1,294 @@ +--- +name: pdf +description: Comprehensive PDF manipulation toolkit for extracting text and tables, creating new PDFs, merging/splitting documents, and handling forms. When Claude needs to fill in a PDF form or programmatically process, generate, or analyze PDF documents at scale. +license: Proprietary. LICENSE.txt has complete terms +--- + +# PDF Processing Guide + +## Overview + +This guide covers essential PDF processing operations using Python libraries and command-line tools. For advanced features, JavaScript libraries, and detailed examples, see reference.md. If you need to fill out a PDF form, read forms.md and follow its instructions. + +## Quick Start + +```python +from pypdf import PdfReader, PdfWriter + +# Read a PDF +reader = PdfReader("document.pdf") +print(f"Pages: {len(reader.pages)}") + +# Extract text +text = "" +for page in reader.pages: + text += page.extract_text() +``` + +## Python Libraries + +### pypdf - Basic Operations + +#### Merge PDFs +```python +from pypdf import PdfWriter, PdfReader + +writer = PdfWriter() +for pdf_file in ["doc1.pdf", "doc2.pdf", "doc3.pdf"]: + reader = PdfReader(pdf_file) + for page in reader.pages: + writer.add_page(page) + +with open("merged.pdf", "wb") as output: + writer.write(output) +``` + +#### Split PDF +```python +reader = PdfReader("input.pdf") +for i, page in enumerate(reader.pages): + writer = PdfWriter() + writer.add_page(page) + with open(f"page_{i+1}.pdf", "wb") as output: + writer.write(output) +``` + +#### Extract Metadata +```python +reader = PdfReader("document.pdf") +meta = reader.metadata +print(f"Title: {meta.title}") +print(f"Author: {meta.author}") +print(f"Subject: {meta.subject}") +print(f"Creator: {meta.creator}") +``` + +#### Rotate Pages +```python +reader = PdfReader("input.pdf") +writer = PdfWriter() + +page = reader.pages[0] +page.rotate(90) # Rotate 90 degrees clockwise +writer.add_page(page) + +with open("rotated.pdf", "wb") as output: + writer.write(output) +``` + +### pdfplumber - Text and Table Extraction + +#### Extract Text with Layout +```python +import pdfplumber + +with pdfplumber.open("document.pdf") as pdf: + for page in pdf.pages: + text = page.extract_text() + print(text) +``` + +#### Extract Tables +```python +with pdfplumber.open("document.pdf") as pdf: + for i, page in enumerate(pdf.pages): + tables = page.extract_tables() + for j, table in enumerate(tables): + print(f"Table {j+1} on page {i+1}:") + for row in table: + print(row) +``` + +#### Advanced Table Extraction +```python +import pandas as pd + +with pdfplumber.open("document.pdf") as pdf: + all_tables = [] + for page in pdf.pages: + tables = page.extract_tables() + for table in tables: + if table: # Check if table is not empty + df = pd.DataFrame(table[1:], columns=table[0]) + all_tables.append(df) + +# Combine all tables +if all_tables: + combined_df = pd.concat(all_tables, ignore_index=True) + combined_df.to_excel("extracted_tables.xlsx", index=False) +``` + +### reportlab - Create PDFs + +#### Basic PDF Creation +```python +from reportlab.lib.pagesizes import letter +from reportlab.pdfgen import canvas + +c = canvas.Canvas("hello.pdf", pagesize=letter) +width, height = letter + +# Add text +c.drawString(100, height - 100, "Hello World!") +c.drawString(100, height - 120, "This is a PDF created with reportlab") + +# Add a line +c.line(100, height - 140, 400, height - 140) + +# Save +c.save() +``` + +#### Create PDF with Multiple Pages +```python +from reportlab.lib.pagesizes import letter +from reportlab.platypus import SimpleDocTemplate, Paragraph, Spacer, PageBreak +from reportlab.lib.styles import getSampleStyleSheet + +doc = SimpleDocTemplate("report.pdf", pagesize=letter) +styles = getSampleStyleSheet() +story = [] + +# Add content +title = Paragraph("Report Title", styles['Title']) +story.append(title) +story.append(Spacer(1, 12)) + +body = Paragraph("This is the body of the report. " * 20, styles['Normal']) +story.append(body) +story.append(PageBreak()) + +# Page 2 +story.append(Paragraph("Page 2", styles['Heading1'])) +story.append(Paragraph("Content for page 2", styles['Normal'])) + +# Build PDF +doc.build(story) +``` + +## Command-Line Tools + +### pdftotext (poppler-utils) +```bash +# Extract text +pdftotext input.pdf output.txt + +# Extract text preserving layout +pdftotext -layout input.pdf output.txt + +# Extract specific pages +pdftotext -f 1 -l 5 input.pdf output.txt # Pages 1-5 +``` + +### qpdf +```bash +# Merge PDFs +qpdf --empty --pages file1.pdf file2.pdf -- merged.pdf + +# Split pages +qpdf input.pdf --pages . 1-5 -- pages1-5.pdf +qpdf input.pdf --pages . 6-10 -- pages6-10.pdf + +# Rotate pages +qpdf input.pdf output.pdf --rotate=+90:1 # Rotate page 1 by 90 degrees + +# Remove password +qpdf --password=mypassword --decrypt encrypted.pdf decrypted.pdf +``` + +### pdftk (if available) +```bash +# Merge +pdftk file1.pdf file2.pdf cat output merged.pdf + +# Split +pdftk input.pdf burst + +# Rotate +pdftk input.pdf rotate 1east output rotated.pdf +``` + +## Common Tasks + +### Extract Text from Scanned PDFs +```python +# Requires: pip install pytesseract pdf2image +import pytesseract +from pdf2image import convert_from_path + +# Convert PDF to images +images = convert_from_path('scanned.pdf') + +# OCR each page +text = "" +for i, image in enumerate(images): + text += f"Page {i+1}:\n" + text += pytesseract.image_to_string(image) + text += "\n\n" + +print(text) +``` + +### Add Watermark +```python +from pypdf import PdfReader, PdfWriter + +# Create watermark (or load existing) +watermark = PdfReader("watermark.pdf").pages[0] + +# Apply to all pages +reader = PdfReader("document.pdf") +writer = PdfWriter() + +for page in reader.pages: + page.merge_page(watermark) + writer.add_page(page) + +with open("watermarked.pdf", "wb") as output: + writer.write(output) +``` + +### Extract Images +```bash +# Using pdfimages (poppler-utils) +pdfimages -j input.pdf output_prefix + +# This extracts all images as output_prefix-000.jpg, output_prefix-001.jpg, etc. +``` + +### Password Protection +```python +from pypdf import PdfReader, PdfWriter + +reader = PdfReader("input.pdf") +writer = PdfWriter() + +for page in reader.pages: + writer.add_page(page) + +# Add password +writer.encrypt("userpassword", "ownerpassword") + +with open("encrypted.pdf", "wb") as output: + writer.write(output) +``` + +## Quick Reference + +| Task | Best Tool | Command/Code | +|------|-----------|--------------| +| Merge PDFs | pypdf | `writer.add_page(page)` | +| Split PDFs | pypdf | One page per file | +| Extract text | pdfplumber | `page.extract_text()` | +| Extract tables | pdfplumber | `page.extract_tables()` | +| Create PDFs | reportlab | Canvas or Platypus | +| Command line merge | qpdf | `qpdf --empty --pages ...` | +| OCR scanned PDFs | pytesseract | Convert to image first | +| Fill PDF forms | pdf-lib or pypdf (see forms.md) | See forms.md | + +## Next Steps + +- For advanced pypdfium2 usage, see reference.md +- For JavaScript libraries (pdf-lib), see reference.md +- If you need to fill out a PDF form, follow the instructions in forms.md +- For troubleshooting guides, see reference.md diff --git a/skills/document-skills/pdf/forms.md b/skills/document-skills/pdf/forms.md new file mode 100644 index 0000000..4e23450 --- /dev/null +++ b/skills/document-skills/pdf/forms.md @@ -0,0 +1,205 @@ +**CRITICAL: You MUST complete these steps in order. Do not skip ahead to writing code.** + +If you need to fill out a PDF form, first check to see if the PDF has fillable form fields. Run this script from this file's directory: + `python scripts/check_fillable_fields `, and depending on the result go to either the "Fillable fields" or "Non-fillable fields" and follow those instructions. + +# Fillable fields +If the PDF has fillable form fields: +- Run this script from this file's directory: `python scripts/extract_form_field_info.py `. It will create a JSON file with a list of fields in this format: +``` +[ + { + "field_id": (unique ID for the field), + "page": (page number, 1-based), + "rect": ([left, bottom, right, top] bounding box in PDF coordinates, y=0 is the bottom of the page), + "type": ("text", "checkbox", "radio_group", or "choice"), + }, + // Checkboxes have "checked_value" and "unchecked_value" properties: + { + "field_id": (unique ID for the field), + "page": (page number, 1-based), + "type": "checkbox", + "checked_value": (Set the field to this value to check the checkbox), + "unchecked_value": (Set the field to this value to uncheck the checkbox), + }, + // Radio groups have a "radio_options" list with the possible choices. + { + "field_id": (unique ID for the field), + "page": (page number, 1-based), + "type": "radio_group", + "radio_options": [ + { + "value": (set the field to this value to select this radio option), + "rect": (bounding box for the radio button for this option) + }, + // Other radio options + ] + }, + // Multiple choice fields have a "choice_options" list with the possible choices: + { + "field_id": (unique ID for the field), + "page": (page number, 1-based), + "type": "choice", + "choice_options": [ + { + "value": (set the field to this value to select this option), + "text": (display text of the option) + }, + // Other choice options + ], + } +] +``` +- Convert the PDF to PNGs (one image for each page) with this script (run from this file's directory): +`python scripts/convert_pdf_to_images.py ` +Then analyze the images to determine the purpose of each form field (make sure to convert the bounding box PDF coordinates to image coordinates). +- Create a `field_values.json` file in this format with the values to be entered for each field: +``` +[ + { + "field_id": "last_name", // Must match the field_id from `extract_form_field_info.py` + "description": "The user's last name", + "page": 1, // Must match the "page" value in field_info.json + "value": "Simpson" + }, + { + "field_id": "Checkbox12", + "description": "Checkbox to be checked if the user is 18 or over", + "page": 1, + "value": "/On" // If this is a checkbox, use its "checked_value" value to check it. If it's a radio button group, use one of the "value" values in "radio_options". + }, + // more fields +] +``` +- Run the `fill_fillable_fields.py` script from this file's directory to create a filled-in PDF: +`python scripts/fill_fillable_fields.py ` +This script will verify that the field IDs and values you provide are valid; if it prints error messages, correct the appropriate fields and try again. + +# Non-fillable fields +If the PDF doesn't have fillable form fields, you'll need to visually determine where the data should be added and create text annotations. Follow the below steps *exactly*. You MUST perform all of these steps to ensure that the the form is accurately completed. Details for each step are below. +- Convert the PDF to PNG images and determine field bounding boxes. +- Create a JSON file with field information and validation images showing the bounding boxes. +- Validate the the bounding boxes. +- Use the bounding boxes to fill in the form. + +## Step 1: Visual Analysis (REQUIRED) +- Convert the PDF to PNG images. Run this script from this file's directory: +`python scripts/convert_pdf_to_images.py ` +The script will create a PNG image for each page in the PDF. +- Carefully examine each PNG image and identify all form fields and areas where the user should enter data. For each form field where the user should enter text, determine bounding boxes for both the form field label, and the area where the user should enter text. The label and entry bounding boxes MUST NOT INTERSECT; the text entry box should only include the area where data should be entered. Usually this area will be immediately to the side, above, or below its label. Entry bounding boxes must be tall and wide enough to contain their text. + +These are some examples of form structures that you might see: + +*Label inside box* +``` +┌────────────────────────┐ +│ Name: │ +└────────────────────────┘ +``` +The input area should be to the right of the "Name" label and extend to the edge of the box. + +*Label before line* +``` +Email: _______________________ +``` +The input area should be above the line and include its entire width. + +*Label under line* +``` +_________________________ +Name +``` +The input area should be above the line and include the entire width of the line. This is common for signature and date fields. + +*Label above line* +``` +Please enter any special requests: +________________________________________________ +``` +The input area should extend from the bottom of the label to the line, and should include the entire width of the line. + +*Checkboxes* +``` +Are you a US citizen? Yes □ No □ +``` +For checkboxes: +- Look for small square boxes (□) - these are the actual checkboxes to target. They may be to the left or right of their labels. +- Distinguish between label text ("Yes", "No") and the clickable checkbox squares. +- The entry bounding box should cover ONLY the small square, not the text label. + +### Step 2: Create fields.json and validation images (REQUIRED) +- Create a file named `fields.json` with information for the form fields and bounding boxes in this format: +``` +{ + "pages": [ + { + "page_number": 1, + "image_width": (first page image width in pixels), + "image_height": (first page image height in pixels), + }, + { + "page_number": 2, + "image_width": (second page image width in pixels), + "image_height": (second page image height in pixels), + } + // additional pages + ], + "form_fields": [ + // Example for a text field. + { + "page_number": 1, + "description": "The user's last name should be entered here", + // Bounding boxes are [left, top, right, bottom]. The bounding boxes for the label and text entry should not overlap. + "field_label": "Last name", + "label_bounding_box": [30, 125, 95, 142], + "entry_bounding_box": [100, 125, 280, 142], + "entry_text": { + "text": "Johnson", // This text will be added as an annotation at the entry_bounding_box location + "font_size": 14, // optional, defaults to 14 + "font_color": "000000", // optional, RRGGBB format, defaults to 000000 (black) + } + }, + // Example for a checkbox. TARGET THE SQUARE for the entry bounding box, NOT THE TEXT + { + "page_number": 2, + "description": "Checkbox that should be checked if the user is over 18", + "entry_bounding_box": [140, 525, 155, 540], // Small box over checkbox square + "field_label": "Yes", + "label_bounding_box": [100, 525, 132, 540], // Box containing "Yes" text + // Use "X" to check a checkbox. + "entry_text": { + "text": "X", + } + } + // additional form field entries + ] +} +``` + +Create validation images by running this script from this file's directory for each page: +`python scripts/create_validation_image.py + +The validation images will have red rectangles where text should be entered, and blue rectangles covering label text. + +### Step 3: Validate Bounding Boxes (REQUIRED) +#### Automated intersection check +- Verify that none of bounding boxes intersect and that the entry bounding boxes are tall enough by checking the fields.json file with the `check_bounding_boxes.py` script (run from this file's directory): +`python scripts/check_bounding_boxes.py ` + +If there are errors, reanalyze the relevant fields, adjust the bounding boxes, and iterate until there are no remaining errors. Remember: label (blue) bounding boxes should contain text labels, entry (red) boxes should not. + +#### Manual image inspection +**CRITICAL: Do not proceed without visually inspecting validation images** +- Red rectangles must ONLY cover input areas +- Red rectangles MUST NOT contain any text +- Blue rectangles should contain label text +- For checkboxes: + - Red rectangle MUST be centered on the checkbox square + - Blue rectangle should cover the text label for the checkbox + +- If any rectangles look wrong, fix fields.json, regenerate the validation images, and verify again. Repeat this process until the bounding boxes are fully accurate. + + +### Step 4: Add annotations to the PDF +Run this script from this file's directory to create a filled-out PDF using the information in fields.json: +`python scripts/fill_pdf_form_with_annotations.py diff --git a/skills/document-skills/pdf/reference.md b/skills/document-skills/pdf/reference.md new file mode 100644 index 0000000..41400bf --- /dev/null +++ b/skills/document-skills/pdf/reference.md @@ -0,0 +1,612 @@ +# PDF Processing Advanced Reference + +This document contains advanced PDF processing features, detailed examples, and additional libraries not covered in the main skill instructions. + +## pypdfium2 Library (Apache/BSD License) + +### Overview +pypdfium2 is a Python binding for PDFium (Chromium's PDF library). It's excellent for fast PDF rendering, image generation, and serves as a PyMuPDF replacement. + +### Render PDF to Images +```python +import pypdfium2 as pdfium +from PIL import Image + +# Load PDF +pdf = pdfium.PdfDocument("document.pdf") + +# Render page to image +page = pdf[0] # First page +bitmap = page.render( + scale=2.0, # Higher resolution + rotation=0 # No rotation +) + +# Convert to PIL Image +img = bitmap.to_pil() +img.save("page_1.png", "PNG") + +# Process multiple pages +for i, page in enumerate(pdf): + bitmap = page.render(scale=1.5) + img = bitmap.to_pil() + img.save(f"page_{i+1}.jpg", "JPEG", quality=90) +``` + +### Extract Text with pypdfium2 +```python +import pypdfium2 as pdfium + +pdf = pdfium.PdfDocument("document.pdf") +for i, page in enumerate(pdf): + text = page.get_text() + print(f"Page {i+1} text length: {len(text)} chars") +``` + +## JavaScript Libraries + +### pdf-lib (MIT License) + +pdf-lib is a powerful JavaScript library for creating and modifying PDF documents in any JavaScript environment. + +#### Load and Manipulate Existing PDF +```javascript +import { PDFDocument } from 'pdf-lib'; +import fs from 'fs'; + +async function manipulatePDF() { + // Load existing PDF + const existingPdfBytes = fs.readFileSync('input.pdf'); + const pdfDoc = await PDFDocument.load(existingPdfBytes); + + // Get page count + const pageCount = pdfDoc.getPageCount(); + console.log(`Document has ${pageCount} pages`); + + // Add new page + const newPage = pdfDoc.addPage([600, 400]); + newPage.drawText('Added by pdf-lib', { + x: 100, + y: 300, + size: 16 + }); + + // Save modified PDF + const pdfBytes = await pdfDoc.save(); + fs.writeFileSync('modified.pdf', pdfBytes); +} +``` + +#### Create Complex PDFs from Scratch +```javascript +import { PDFDocument, rgb, StandardFonts } from 'pdf-lib'; +import fs from 'fs'; + +async function createPDF() { + const pdfDoc = await PDFDocument.create(); + + // Add fonts + const helveticaFont = await pdfDoc.embedFont(StandardFonts.Helvetica); + const helveticaBold = await pdfDoc.embedFont(StandardFonts.HelveticaBold); + + // Add page + const page = pdfDoc.addPage([595, 842]); // A4 size + const { width, height } = page.getSize(); + + // Add text with styling + page.drawText('Invoice #12345', { + x: 50, + y: height - 50, + size: 18, + font: helveticaBold, + color: rgb(0.2, 0.2, 0.8) + }); + + // Add rectangle (header background) + page.drawRectangle({ + x: 40, + y: height - 100, + width: width - 80, + height: 30, + color: rgb(0.9, 0.9, 0.9) + }); + + // Add table-like content + const items = [ + ['Item', 'Qty', 'Price', 'Total'], + ['Widget', '2', '$50', '$100'], + ['Gadget', '1', '$75', '$75'] + ]; + + let yPos = height - 150; + items.forEach(row => { + let xPos = 50; + row.forEach(cell => { + page.drawText(cell, { + x: xPos, + y: yPos, + size: 12, + font: helveticaFont + }); + xPos += 120; + }); + yPos -= 25; + }); + + const pdfBytes = await pdfDoc.save(); + fs.writeFileSync('created.pdf', pdfBytes); +} +``` + +#### Advanced Merge and Split Operations +```javascript +import { PDFDocument } from 'pdf-lib'; +import fs from 'fs'; + +async function mergePDFs() { + // Create new document + const mergedPdf = await PDFDocument.create(); + + // Load source PDFs + const pdf1Bytes = fs.readFileSync('doc1.pdf'); + const pdf2Bytes = fs.readFileSync('doc2.pdf'); + + const pdf1 = await PDFDocument.load(pdf1Bytes); + const pdf2 = await PDFDocument.load(pdf2Bytes); + + // Copy pages from first PDF + const pdf1Pages = await mergedPdf.copyPages(pdf1, pdf1.getPageIndices()); + pdf1Pages.forEach(page => mergedPdf.addPage(page)); + + // Copy specific pages from second PDF (pages 0, 2, 4) + const pdf2Pages = await mergedPdf.copyPages(pdf2, [0, 2, 4]); + pdf2Pages.forEach(page => mergedPdf.addPage(page)); + + const mergedPdfBytes = await mergedPdf.save(); + fs.writeFileSync('merged.pdf', mergedPdfBytes); +} +``` + +### pdfjs-dist (Apache License) + +PDF.js is Mozilla's JavaScript library for rendering PDFs in the browser. + +#### Basic PDF Loading and Rendering +```javascript +import * as pdfjsLib from 'pdfjs-dist'; + +// Configure worker (important for performance) +pdfjsLib.GlobalWorkerOptions.workerSrc = './pdf.worker.js'; + +async function renderPDF() { + // Load PDF + const loadingTask = pdfjsLib.getDocument('document.pdf'); + const pdf = await loadingTask.promise; + + console.log(`Loaded PDF with ${pdf.numPages} pages`); + + // Get first page + const page = await pdf.getPage(1); + const viewport = page.getViewport({ scale: 1.5 }); + + // Render to canvas + const canvas = document.createElement('canvas'); + const context = canvas.getContext('2d'); + canvas.height = viewport.height; + canvas.width = viewport.width; + + const renderContext = { + canvasContext: context, + viewport: viewport + }; + + await page.render(renderContext).promise; + document.body.appendChild(canvas); +} +``` + +#### Extract Text with Coordinates +```javascript +import * as pdfjsLib from 'pdfjs-dist'; + +async function extractText() { + const loadingTask = pdfjsLib.getDocument('document.pdf'); + const pdf = await loadingTask.promise; + + let fullText = ''; + + // Extract text from all pages + for (let i = 1; i <= pdf.numPages; i++) { + const page = await pdf.getPage(i); + const textContent = await page.getTextContent(); + + const pageText = textContent.items + .map(item => item.str) + .join(' '); + + fullText += `\n--- Page ${i} ---\n${pageText}`; + + // Get text with coordinates for advanced processing + const textWithCoords = textContent.items.map(item => ({ + text: item.str, + x: item.transform[4], + y: item.transform[5], + width: item.width, + height: item.height + })); + } + + console.log(fullText); + return fullText; +} +``` + +#### Extract Annotations and Forms +```javascript +import * as pdfjsLib from 'pdfjs-dist'; + +async function extractAnnotations() { + const loadingTask = pdfjsLib.getDocument('annotated.pdf'); + const pdf = await loadingTask.promise; + + for (let i = 1; i <= pdf.numPages; i++) { + const page = await pdf.getPage(i); + const annotations = await page.getAnnotations(); + + annotations.forEach(annotation => { + console.log(`Annotation type: ${annotation.subtype}`); + console.log(`Content: ${annotation.contents}`); + console.log(`Coordinates: ${JSON.stringify(annotation.rect)}`); + }); + } +} +``` + +## Advanced Command-Line Operations + +### poppler-utils Advanced Features + +#### Extract Text with Bounding Box Coordinates +```bash +# Extract text with bounding box coordinates (essential for structured data) +pdftotext -bbox-layout document.pdf output.xml + +# The XML output contains precise coordinates for each text element +``` + +#### Advanced Image Conversion +```bash +# Convert to PNG images with specific resolution +pdftoppm -png -r 300 document.pdf output_prefix + +# Convert specific page range with high resolution +pdftoppm -png -r 600 -f 1 -l 3 document.pdf high_res_pages + +# Convert to JPEG with quality setting +pdftoppm -jpeg -jpegopt quality=85 -r 200 document.pdf jpeg_output +``` + +#### Extract Embedded Images +```bash +# Extract all embedded images with metadata +pdfimages -j -p document.pdf page_images + +# List image info without extracting +pdfimages -list document.pdf + +# Extract images in their original format +pdfimages -all document.pdf images/img +``` + +### qpdf Advanced Features + +#### Complex Page Manipulation +```bash +# Split PDF into groups of pages +qpdf --split-pages=3 input.pdf output_group_%02d.pdf + +# Extract specific pages with complex ranges +qpdf input.pdf --pages input.pdf 1,3-5,8,10-end -- extracted.pdf + +# Merge specific pages from multiple PDFs +qpdf --empty --pages doc1.pdf 1-3 doc2.pdf 5-7 doc3.pdf 2,4 -- combined.pdf +``` + +#### PDF Optimization and Repair +```bash +# Optimize PDF for web (linearize for streaming) +qpdf --linearize input.pdf optimized.pdf + +# Remove unused objects and compress +qpdf --optimize-level=all input.pdf compressed.pdf + +# Attempt to repair corrupted PDF structure +qpdf --check input.pdf +qpdf --fix-qdf damaged.pdf repaired.pdf + +# Show detailed PDF structure for debugging +qpdf --show-all-pages input.pdf > structure.txt +``` + +#### Advanced Encryption +```bash +# Add password protection with specific permissions +qpdf --encrypt user_pass owner_pass 256 --print=none --modify=none -- input.pdf encrypted.pdf + +# Check encryption status +qpdf --show-encryption encrypted.pdf + +# Remove password protection (requires password) +qpdf --password=secret123 --decrypt encrypted.pdf decrypted.pdf +``` + +## Advanced Python Techniques + +### pdfplumber Advanced Features + +#### Extract Text with Precise Coordinates +```python +import pdfplumber + +with pdfplumber.open("document.pdf") as pdf: + page = pdf.pages[0] + + # Extract all text with coordinates + chars = page.chars + for char in chars[:10]: # First 10 characters + print(f"Char: '{char['text']}' at x:{char['x0']:.1f} y:{char['y0']:.1f}") + + # Extract text by bounding box (left, top, right, bottom) + bbox_text = page.within_bbox((100, 100, 400, 200)).extract_text() +``` + +#### Advanced Table Extraction with Custom Settings +```python +import pdfplumber +import pandas as pd + +with pdfplumber.open("complex_table.pdf") as pdf: + page = pdf.pages[0] + + # Extract tables with custom settings for complex layouts + table_settings = { + "vertical_strategy": "lines", + "horizontal_strategy": "lines", + "snap_tolerance": 3, + "intersection_tolerance": 15 + } + tables = page.extract_tables(table_settings) + + # Visual debugging for table extraction + img = page.to_image(resolution=150) + img.save("debug_layout.png") +``` + +### reportlab Advanced Features + +#### Create Professional Reports with Tables +```python +from reportlab.platypus import SimpleDocTemplate, Table, TableStyle, Paragraph +from reportlab.lib.styles import getSampleStyleSheet +from reportlab.lib import colors + +# Sample data +data = [ + ['Product', 'Q1', 'Q2', 'Q3', 'Q4'], + ['Widgets', '120', '135', '142', '158'], + ['Gadgets', '85', '92', '98', '105'] +] + +# Create PDF with table +doc = SimpleDocTemplate("report.pdf") +elements = [] + +# Add title +styles = getSampleStyleSheet() +title = Paragraph("Quarterly Sales Report", styles['Title']) +elements.append(title) + +# Add table with advanced styling +table = Table(data) +table.setStyle(TableStyle([ + ('BACKGROUND', (0, 0), (-1, 0), colors.grey), + ('TEXTCOLOR', (0, 0), (-1, 0), colors.whitesmoke), + ('ALIGN', (0, 0), (-1, -1), 'CENTER'), + ('FONTNAME', (0, 0), (-1, 0), 'Helvetica-Bold'), + ('FONTSIZE', (0, 0), (-1, 0), 14), + ('BOTTOMPADDING', (0, 0), (-1, 0), 12), + ('BACKGROUND', (0, 1), (-1, -1), colors.beige), + ('GRID', (0, 0), (-1, -1), 1, colors.black) +])) +elements.append(table) + +doc.build(elements) +``` + +## Complex Workflows + +### Extract Figures/Images from PDF + +#### Method 1: Using pdfimages (fastest) +```bash +# Extract all images with original quality +pdfimages -all document.pdf images/img +``` + +#### Method 2: Using pypdfium2 + Image Processing +```python +import pypdfium2 as pdfium +from PIL import Image +import numpy as np + +def extract_figures(pdf_path, output_dir): + pdf = pdfium.PdfDocument(pdf_path) + + for page_num, page in enumerate(pdf): + # Render high-resolution page + bitmap = page.render(scale=3.0) + img = bitmap.to_pil() + + # Convert to numpy for processing + img_array = np.array(img) + + # Simple figure detection (non-white regions) + mask = np.any(img_array != [255, 255, 255], axis=2) + + # Find contours and extract bounding boxes + # (This is simplified - real implementation would need more sophisticated detection) + + # Save detected figures + # ... implementation depends on specific needs +``` + +### Batch PDF Processing with Error Handling +```python +import os +import glob +from pypdf import PdfReader, PdfWriter +import logging + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + +def batch_process_pdfs(input_dir, operation='merge'): + pdf_files = glob.glob(os.path.join(input_dir, "*.pdf")) + + if operation == 'merge': + writer = PdfWriter() + for pdf_file in pdf_files: + try: + reader = PdfReader(pdf_file) + for page in reader.pages: + writer.add_page(page) + logger.info(f"Processed: {pdf_file}") + except Exception as e: + logger.error(f"Failed to process {pdf_file}: {e}") + continue + + with open("batch_merged.pdf", "wb") as output: + writer.write(output) + + elif operation == 'extract_text': + for pdf_file in pdf_files: + try: + reader = PdfReader(pdf_file) + text = "" + for page in reader.pages: + text += page.extract_text() + + output_file = pdf_file.replace('.pdf', '.txt') + with open(output_file, 'w', encoding='utf-8') as f: + f.write(text) + logger.info(f"Extracted text from: {pdf_file}") + + except Exception as e: + logger.error(f"Failed to extract text from {pdf_file}: {e}") + continue +``` + +### Advanced PDF Cropping +```python +from pypdf import PdfWriter, PdfReader + +reader = PdfReader("input.pdf") +writer = PdfWriter() + +# Crop page (left, bottom, right, top in points) +page = reader.pages[0] +page.mediabox.left = 50 +page.mediabox.bottom = 50 +page.mediabox.right = 550 +page.mediabox.top = 750 + +writer.add_page(page) +with open("cropped.pdf", "wb") as output: + writer.write(output) +``` + +## Performance Optimization Tips + +### 1. For Large PDFs +- Use streaming approaches instead of loading entire PDF in memory +- Use `qpdf --split-pages` for splitting large files +- Process pages individually with pypdfium2 + +### 2. For Text Extraction +- `pdftotext -bbox-layout` is fastest for plain text extraction +- Use pdfplumber for structured data and tables +- Avoid `pypdf.extract_text()` for very large documents + +### 3. For Image Extraction +- `pdfimages` is much faster than rendering pages +- Use low resolution for previews, high resolution for final output + +### 4. For Form Filling +- pdf-lib maintains form structure better than most alternatives +- Pre-validate form fields before processing + +### 5. Memory Management +```python +# Process PDFs in chunks +def process_large_pdf(pdf_path, chunk_size=10): + reader = PdfReader(pdf_path) + total_pages = len(reader.pages) + + for start_idx in range(0, total_pages, chunk_size): + end_idx = min(start_idx + chunk_size, total_pages) + writer = PdfWriter() + + for i in range(start_idx, end_idx): + writer.add_page(reader.pages[i]) + + # Process chunk + with open(f"chunk_{start_idx//chunk_size}.pdf", "wb") as output: + writer.write(output) +``` + +## Troubleshooting Common Issues + +### Encrypted PDFs +```python +# Handle password-protected PDFs +from pypdf import PdfReader + +try: + reader = PdfReader("encrypted.pdf") + if reader.is_encrypted: + reader.decrypt("password") +except Exception as e: + print(f"Failed to decrypt: {e}") +``` + +### Corrupted PDFs +```bash +# Use qpdf to repair +qpdf --check corrupted.pdf +qpdf --replace-input corrupted.pdf +``` + +### Text Extraction Issues +```python +# Fallback to OCR for scanned PDFs +import pytesseract +from pdf2image import convert_from_path + +def extract_text_with_ocr(pdf_path): + images = convert_from_path(pdf_path) + text = "" + for i, image in enumerate(images): + text += pytesseract.image_to_string(image) + return text +``` + +## License Information + +- **pypdf**: BSD License +- **pdfplumber**: MIT License +- **pypdfium2**: Apache/BSD License +- **reportlab**: BSD License +- **poppler-utils**: GPL-2 License +- **qpdf**: Apache License +- **pdf-lib**: MIT License +- **pdfjs-dist**: Apache License \ No newline at end of file diff --git a/skills/document-skills/pdf/scripts/check_bounding_boxes.py b/skills/document-skills/pdf/scripts/check_bounding_boxes.py new file mode 100644 index 0000000..7443660 --- /dev/null +++ b/skills/document-skills/pdf/scripts/check_bounding_boxes.py @@ -0,0 +1,70 @@ +from dataclasses import dataclass +import json +import sys + + +# Script to check that the `fields.json` file that Claude creates when analyzing PDFs +# does not have overlapping bounding boxes. See forms.md. + + +@dataclass +class RectAndField: + rect: list[float] + rect_type: str + field: dict + + +# Returns a list of messages that are printed to stdout for Claude to read. +def get_bounding_box_messages(fields_json_stream) -> list[str]: + messages = [] + fields = json.load(fields_json_stream) + messages.append(f"Read {len(fields['form_fields'])} fields") + + def rects_intersect(r1, r2): + disjoint_horizontal = r1[0] >= r2[2] or r1[2] <= r2[0] + disjoint_vertical = r1[1] >= r2[3] or r1[3] <= r2[1] + return not (disjoint_horizontal or disjoint_vertical) + + rects_and_fields = [] + for f in fields["form_fields"]: + rects_and_fields.append(RectAndField(f["label_bounding_box"], "label", f)) + rects_and_fields.append(RectAndField(f["entry_bounding_box"], "entry", f)) + + has_error = False + for i, ri in enumerate(rects_and_fields): + # This is O(N^2); we can optimize if it becomes a problem. + for j in range(i + 1, len(rects_and_fields)): + rj = rects_and_fields[j] + if ri.field["page_number"] == rj.field["page_number"] and rects_intersect(ri.rect, rj.rect): + has_error = True + if ri.field is rj.field: + messages.append(f"FAILURE: intersection between label and entry bounding boxes for `{ri.field['description']}` ({ri.rect}, {rj.rect})") + else: + messages.append(f"FAILURE: intersection between {ri.rect_type} bounding box for `{ri.field['description']}` ({ri.rect}) and {rj.rect_type} bounding box for `{rj.field['description']}` ({rj.rect})") + if len(messages) >= 20: + messages.append("Aborting further checks; fix bounding boxes and try again") + return messages + if ri.rect_type == "entry": + if "entry_text" in ri.field: + font_size = ri.field["entry_text"].get("font_size", 14) + entry_height = ri.rect[3] - ri.rect[1] + if entry_height < font_size: + has_error = True + messages.append(f"FAILURE: entry bounding box height ({entry_height}) for `{ri.field['description']}` is too short for the text content (font size: {font_size}). Increase the box height or decrease the font size.") + if len(messages) >= 20: + messages.append("Aborting further checks; fix bounding boxes and try again") + return messages + + if not has_error: + messages.append("SUCCESS: All bounding boxes are valid") + return messages + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: check_bounding_boxes.py [fields.json]") + sys.exit(1) + # Input file should be in the `fields.json` format described in forms.md. + with open(sys.argv[1]) as f: + messages = get_bounding_box_messages(f) + for msg in messages: + print(msg) diff --git a/skills/document-skills/pdf/scripts/check_bounding_boxes_test.py b/skills/document-skills/pdf/scripts/check_bounding_boxes_test.py new file mode 100644 index 0000000..1dbb463 --- /dev/null +++ b/skills/document-skills/pdf/scripts/check_bounding_boxes_test.py @@ -0,0 +1,226 @@ +import unittest +import json +import io +from check_bounding_boxes import get_bounding_box_messages + + +# Currently this is not run automatically in CI; it's just for documentation and manual checking. +class TestGetBoundingBoxMessages(unittest.TestCase): + + def create_json_stream(self, data): + """Helper to create a JSON stream from data""" + return io.StringIO(json.dumps(data)) + + def test_no_intersections(self): + """Test case with no bounding box intersections""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30] + }, + { + "description": "Email", + "page_number": 1, + "label_bounding_box": [10, 40, 50, 60], + "entry_bounding_box": [60, 40, 150, 60] + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_label_entry_intersection_same_field(self): + """Test intersection between label and entry of the same field""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 60, 30], + "entry_bounding_box": [50, 10, 150, 30] # Overlaps with label + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "intersection" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_intersection_between_different_fields(self): + """Test intersection between bounding boxes of different fields""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30] + }, + { + "description": "Email", + "page_number": 1, + "label_bounding_box": [40, 20, 80, 40], # Overlaps with Name's boxes + "entry_bounding_box": [160, 10, 250, 30] + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "intersection" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_different_pages_no_intersection(self): + """Test that boxes on different pages don't count as intersecting""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30] + }, + { + "description": "Email", + "page_number": 2, + "label_bounding_box": [10, 10, 50, 30], # Same coordinates but different page + "entry_bounding_box": [60, 10, 150, 30] + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_entry_height_too_small(self): + """Test that entry box height is checked against font size""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 20], # Height is 10 + "entry_text": { + "font_size": 14 # Font size larger than height + } + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "height" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_entry_height_adequate(self): + """Test that adequate entry box height passes""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 30], # Height is 20 + "entry_text": { + "font_size": 14 # Font size smaller than height + } + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_default_font_size(self): + """Test that default font size is used when not specified""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 20], # Height is 10 + "entry_text": {} # No font_size specified, should use default 14 + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("FAILURE" in msg and "height" in msg for msg in messages)) + self.assertFalse(any("SUCCESS" in msg for msg in messages)) + + def test_no_entry_text(self): + """Test that missing entry_text doesn't cause height check""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [60, 10, 150, 20] # Small height but no entry_text + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + def test_multiple_errors_limit(self): + """Test that error messages are limited to prevent excessive output""" + fields = [] + # Create many overlapping fields + for i in range(25): + fields.append({ + "description": f"Field{i}", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], # All overlap + "entry_bounding_box": [20, 15, 60, 35] # All overlap + }) + + data = {"form_fields": fields} + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + # Should abort after ~20 messages + self.assertTrue(any("Aborting" in msg for msg in messages)) + # Should have some FAILURE messages but not hundreds + failure_count = sum(1 for msg in messages if "FAILURE" in msg) + self.assertGreater(failure_count, 0) + self.assertLess(len(messages), 30) # Should be limited + + def test_edge_touching_boxes(self): + """Test that boxes touching at edges don't count as intersecting""" + data = { + "form_fields": [ + { + "description": "Name", + "page_number": 1, + "label_bounding_box": [10, 10, 50, 30], + "entry_bounding_box": [50, 10, 150, 30] # Touches at x=50 + } + ] + } + + stream = self.create_json_stream(data) + messages = get_bounding_box_messages(stream) + self.assertTrue(any("SUCCESS" in msg for msg in messages)) + self.assertFalse(any("FAILURE" in msg for msg in messages)) + + +if __name__ == '__main__': + unittest.main() diff --git a/skills/document-skills/pdf/scripts/check_fillable_fields.py b/skills/document-skills/pdf/scripts/check_fillable_fields.py new file mode 100644 index 0000000..dc43d18 --- /dev/null +++ b/skills/document-skills/pdf/scripts/check_fillable_fields.py @@ -0,0 +1,12 @@ +import sys +from pypdf import PdfReader + + +# Script for Claude to run to determine whether a PDF has fillable form fields. See forms.md. + + +reader = PdfReader(sys.argv[1]) +if (reader.get_fields()): + print("This PDF has fillable form fields") +else: + print("This PDF does not have fillable form fields; you will need to visually determine where to enter data") diff --git a/skills/document-skills/pdf/scripts/convert_pdf_to_images.py b/skills/document-skills/pdf/scripts/convert_pdf_to_images.py new file mode 100644 index 0000000..f8a4ec5 --- /dev/null +++ b/skills/document-skills/pdf/scripts/convert_pdf_to_images.py @@ -0,0 +1,35 @@ +import os +import sys + +from pdf2image import convert_from_path + + +# Converts each page of a PDF to a PNG image. + + +def convert(pdf_path, output_dir, max_dim=1000): + images = convert_from_path(pdf_path, dpi=200) + + for i, image in enumerate(images): + # Scale image if needed to keep width/height under `max_dim` + width, height = image.size + if width > max_dim or height > max_dim: + scale_factor = min(max_dim / width, max_dim / height) + new_width = int(width * scale_factor) + new_height = int(height * scale_factor) + image = image.resize((new_width, new_height)) + + image_path = os.path.join(output_dir, f"page_{i+1}.png") + image.save(image_path) + print(f"Saved page {i+1} as {image_path} (size: {image.size})") + + print(f"Converted {len(images)} pages to PNG images") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: convert_pdf_to_images.py [input pdf] [output directory]") + sys.exit(1) + pdf_path = sys.argv[1] + output_directory = sys.argv[2] + convert(pdf_path, output_directory) diff --git a/skills/document-skills/pdf/scripts/create_validation_image.py b/skills/document-skills/pdf/scripts/create_validation_image.py new file mode 100644 index 0000000..4913f8f --- /dev/null +++ b/skills/document-skills/pdf/scripts/create_validation_image.py @@ -0,0 +1,41 @@ +import json +import sys + +from PIL import Image, ImageDraw + + +# Creates "validation" images with rectangles for the bounding box information that +# Claude creates when determining where to add text annotations in PDFs. See forms.md. + + +def create_validation_image(page_number, fields_json_path, input_path, output_path): + # Input file should be in the `fields.json` format described in forms.md. + with open(fields_json_path, 'r') as f: + data = json.load(f) + + img = Image.open(input_path) + draw = ImageDraw.Draw(img) + num_boxes = 0 + + for field in data["form_fields"]: + if field["page_number"] == page_number: + entry_box = field['entry_bounding_box'] + label_box = field['label_bounding_box'] + # Draw red rectangle over entry bounding box and blue rectangle over the label. + draw.rectangle(entry_box, outline='red', width=2) + draw.rectangle(label_box, outline='blue', width=2) + num_boxes += 2 + + img.save(output_path) + print(f"Created validation image at {output_path} with {num_boxes} bounding boxes") + + +if __name__ == "__main__": + if len(sys.argv) != 5: + print("Usage: create_validation_image.py [page number] [fields.json file] [input image path] [output image path]") + sys.exit(1) + page_number = int(sys.argv[1]) + fields_json_path = sys.argv[2] + input_image_path = sys.argv[3] + output_image_path = sys.argv[4] + create_validation_image(page_number, fields_json_path, input_image_path, output_image_path) diff --git a/skills/document-skills/pdf/scripts/extract_form_field_info.py b/skills/document-skills/pdf/scripts/extract_form_field_info.py new file mode 100644 index 0000000..f42a2df --- /dev/null +++ b/skills/document-skills/pdf/scripts/extract_form_field_info.py @@ -0,0 +1,152 @@ +import json +import sys + +from pypdf import PdfReader + + +# Extracts data for the fillable form fields in a PDF and outputs JSON that +# Claude uses to fill the fields. See forms.md. + + +# This matches the format used by PdfReader `get_fields` and `update_page_form_field_values` methods. +def get_full_annotation_field_id(annotation): + components = [] + while annotation: + field_name = annotation.get('/T') + if field_name: + components.append(field_name) + annotation = annotation.get('/Parent') + return ".".join(reversed(components)) if components else None + + +def make_field_dict(field, field_id): + field_dict = {"field_id": field_id} + ft = field.get('/FT') + if ft == "/Tx": + field_dict["type"] = "text" + elif ft == "/Btn": + field_dict["type"] = "checkbox" # radio groups handled separately + states = field.get("/_States_", []) + if len(states) == 2: + # "/Off" seems to always be the unchecked value, as suggested by + # https://opensource.adobe.com/dc-acrobat-sdk-docs/standards/pdfstandards/pdf/PDF32000_2008.pdf#page=448 + # It can be either first or second in the "/_States_" list. + if "/Off" in states: + field_dict["checked_value"] = states[0] if states[0] != "/Off" else states[1] + field_dict["unchecked_value"] = "/Off" + else: + print(f"Unexpected state values for checkbox `${field_id}`. Its checked and unchecked values may not be correct; if you're trying to check it, visually verify the results.") + field_dict["checked_value"] = states[0] + field_dict["unchecked_value"] = states[1] + elif ft == "/Ch": + field_dict["type"] = "choice" + states = field.get("/_States_", []) + field_dict["choice_options"] = [{ + "value": state[0], + "text": state[1], + } for state in states] + else: + field_dict["type"] = f"unknown ({ft})" + return field_dict + + +# Returns a list of fillable PDF fields: +# [ +# { +# "field_id": "name", +# "page": 1, +# "type": ("text", "checkbox", "radio_group", or "choice") +# // Per-type additional fields described in forms.md +# }, +# ] +def get_field_info(reader: PdfReader): + fields = reader.get_fields() + + field_info_by_id = {} + possible_radio_names = set() + + for field_id, field in fields.items(): + # Skip if this is a container field with children, except that it might be + # a parent group for radio button options. + if field.get("/Kids"): + if field.get("/FT") == "/Btn": + possible_radio_names.add(field_id) + continue + field_info_by_id[field_id] = make_field_dict(field, field_id) + + # Bounding rects are stored in annotations in page objects. + + # Radio button options have a separate annotation for each choice; + # all choices have the same field name. + # See https://westhealth.github.io/exploring-fillable-forms-with-pdfrw.html + radio_fields_by_id = {} + + for page_index, page in enumerate(reader.pages): + annotations = page.get('/Annots', []) + for ann in annotations: + field_id = get_full_annotation_field_id(ann) + if field_id in field_info_by_id: + field_info_by_id[field_id]["page"] = page_index + 1 + field_info_by_id[field_id]["rect"] = ann.get('/Rect') + elif field_id in possible_radio_names: + try: + # ann['/AP']['/N'] should have two items. One of them is '/Off', + # the other is the active value. + on_values = [v for v in ann["/AP"]["/N"] if v != "/Off"] + except KeyError: + continue + if len(on_values) == 1: + rect = ann.get("/Rect") + if field_id not in radio_fields_by_id: + radio_fields_by_id[field_id] = { + "field_id": field_id, + "type": "radio_group", + "page": page_index + 1, + "radio_options": [], + } + # Note: at least on macOS 15.7, Preview.app doesn't show selected + # radio buttons correctly. (It does if you remove the leading slash + # from the value, but that causes them not to appear correctly in + # Chrome/Firefox/Acrobat/etc). + radio_fields_by_id[field_id]["radio_options"].append({ + "value": on_values[0], + "rect": rect, + }) + + # Some PDFs have form field definitions without corresponding annotations, + # so we can't tell where they are. Ignore these fields for now. + fields_with_location = [] + for field_info in field_info_by_id.values(): + if "page" in field_info: + fields_with_location.append(field_info) + else: + print(f"Unable to determine location for field id: {field_info.get('field_id')}, ignoring") + + # Sort by page number, then Y position (flipped in PDF coordinate system), then X. + def sort_key(f): + if "radio_options" in f: + rect = f["radio_options"][0]["rect"] or [0, 0, 0, 0] + else: + rect = f.get("rect") or [0, 0, 0, 0] + adjusted_position = [-rect[1], rect[0]] + return [f.get("page"), adjusted_position] + + sorted_fields = fields_with_location + list(radio_fields_by_id.values()) + sorted_fields.sort(key=sort_key) + + return sorted_fields + + +def write_field_info(pdf_path: str, json_output_path: str): + reader = PdfReader(pdf_path) + field_info = get_field_info(reader) + with open(json_output_path, "w") as f: + json.dump(field_info, f, indent=2) + print(f"Wrote {len(field_info)} fields to {json_output_path}") + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print("Usage: extract_form_field_info.py [input pdf] [output json]") + sys.exit(1) + write_field_info(sys.argv[1], sys.argv[2]) diff --git a/skills/document-skills/pdf/scripts/fill_fillable_fields.py b/skills/document-skills/pdf/scripts/fill_fillable_fields.py new file mode 100644 index 0000000..ac35753 --- /dev/null +++ b/skills/document-skills/pdf/scripts/fill_fillable_fields.py @@ -0,0 +1,114 @@ +import json +import sys + +from pypdf import PdfReader, PdfWriter + +from extract_form_field_info import get_field_info + + +# Fills fillable form fields in a PDF. See forms.md. + + +def fill_pdf_fields(input_pdf_path: str, fields_json_path: str, output_pdf_path: str): + with open(fields_json_path) as f: + fields = json.load(f) + # Group by page number. + fields_by_page = {} + for field in fields: + if "value" in field: + field_id = field["field_id"] + page = field["page"] + if page not in fields_by_page: + fields_by_page[page] = {} + fields_by_page[page][field_id] = field["value"] + + reader = PdfReader(input_pdf_path) + + has_error = False + field_info = get_field_info(reader) + fields_by_ids = {f["field_id"]: f for f in field_info} + for field in fields: + existing_field = fields_by_ids.get(field["field_id"]) + if not existing_field: + has_error = True + print(f"ERROR: `{field['field_id']}` is not a valid field ID") + elif field["page"] != existing_field["page"]: + has_error = True + print(f"ERROR: Incorrect page number for `{field['field_id']}` (got {field['page']}, expected {existing_field['page']})") + else: + if "value" in field: + err = validation_error_for_field_value(existing_field, field["value"]) + if err: + print(err) + has_error = True + if has_error: + sys.exit(1) + + writer = PdfWriter(clone_from=reader) + for page, field_values in fields_by_page.items(): + writer.update_page_form_field_values(writer.pages[page - 1], field_values, auto_regenerate=False) + + # This seems to be necessary for many PDF viewers to format the form values correctly. + # It may cause the viewer to show a "save changes" dialog even if the user doesn't make any changes. + writer.set_need_appearances_writer(True) + + with open(output_pdf_path, "wb") as f: + writer.write(f) + + +def validation_error_for_field_value(field_info, field_value): + field_type = field_info["type"] + field_id = field_info["field_id"] + if field_type == "checkbox": + checked_val = field_info["checked_value"] + unchecked_val = field_info["unchecked_value"] + if field_value != checked_val and field_value != unchecked_val: + return f'ERROR: Invalid value "{field_value}" for checkbox field "{field_id}". The checked value is "{checked_val}" and the unchecked value is "{unchecked_val}"' + elif field_type == "radio_group": + option_values = [opt["value"] for opt in field_info["radio_options"]] + if field_value not in option_values: + return f'ERROR: Invalid value "{field_value}" for radio group field "{field_id}". Valid values are: {option_values}' + elif field_type == "choice": + choice_values = [opt["value"] for opt in field_info["choice_options"]] + if field_value not in choice_values: + return f'ERROR: Invalid value "{field_value}" for choice field "{field_id}". Valid values are: {choice_values}' + return None + + +# pypdf (at least version 5.7.0) has a bug when setting the value for a selection list field. +# In _writer.py around line 966: +# +# if field.get(FA.FT, "/Tx") == "/Ch" and field_flags & FA.FfBits.Combo == 0: +# txt = "\n".join(annotation.get_inherited(FA.Opt, [])) +# +# The problem is that for selection lists, `get_inherited` returns a list of two-element lists like +# [["value1", "Text 1"], ["value2", "Text 2"], ...] +# This causes `join` to throw a TypeError because it expects an iterable of strings. +# The horrible workaround is to patch `get_inherited` to return a list of the value strings. +# We call the original method and adjust the return value only if the argument to `get_inherited` +# is `FA.Opt` and if the return value is a list of two-element lists. +def monkeypatch_pydpf_method(): + from pypdf.generic import DictionaryObject + from pypdf.constants import FieldDictionaryAttributes + + original_get_inherited = DictionaryObject.get_inherited + + def patched_get_inherited(self, key: str, default = None): + result = original_get_inherited(self, key, default) + if key == FieldDictionaryAttributes.Opt: + if isinstance(result, list) and all(isinstance(v, list) and len(v) == 2 for v in result): + result = [r[0] for r in result] + return result + + DictionaryObject.get_inherited = patched_get_inherited + + +if __name__ == "__main__": + if len(sys.argv) != 4: + print("Usage: fill_fillable_fields.py [input pdf] [field_values.json] [output pdf]") + sys.exit(1) + monkeypatch_pydpf_method() + input_pdf = sys.argv[1] + fields_json = sys.argv[2] + output_pdf = sys.argv[3] + fill_pdf_fields(input_pdf, fields_json, output_pdf) diff --git a/skills/document-skills/pdf/scripts/fill_pdf_form_with_annotations.py b/skills/document-skills/pdf/scripts/fill_pdf_form_with_annotations.py new file mode 100644 index 0000000..f980531 --- /dev/null +++ b/skills/document-skills/pdf/scripts/fill_pdf_form_with_annotations.py @@ -0,0 +1,108 @@ +import json +import sys + +from pypdf import PdfReader, PdfWriter +from pypdf.annotations import FreeText + + +# Fills a PDF by adding text annotations defined in `fields.json`. See forms.md. + + +def transform_coordinates(bbox, image_width, image_height, pdf_width, pdf_height): + """Transform bounding box from image coordinates to PDF coordinates""" + # Image coordinates: origin at top-left, y increases downward + # PDF coordinates: origin at bottom-left, y increases upward + x_scale = pdf_width / image_width + y_scale = pdf_height / image_height + + left = bbox[0] * x_scale + right = bbox[2] * x_scale + + # Flip Y coordinates for PDF + top = pdf_height - (bbox[1] * y_scale) + bottom = pdf_height - (bbox[3] * y_scale) + + return left, bottom, right, top + + +def fill_pdf_form(input_pdf_path, fields_json_path, output_pdf_path): + """Fill the PDF form with data from fields.json""" + + # `fields.json` format described in forms.md. + with open(fields_json_path, "r") as f: + fields_data = json.load(f) + + # Open the PDF + reader = PdfReader(input_pdf_path) + writer = PdfWriter() + + # Copy all pages to writer + writer.append(reader) + + # Get PDF dimensions for each page + pdf_dimensions = {} + for i, page in enumerate(reader.pages): + mediabox = page.mediabox + pdf_dimensions[i + 1] = [mediabox.width, mediabox.height] + + # Process each form field + annotations = [] + for field in fields_data["form_fields"]: + page_num = field["page_number"] + + # Get page dimensions and transform coordinates. + page_info = next(p for p in fields_data["pages"] if p["page_number"] == page_num) + image_width = page_info["image_width"] + image_height = page_info["image_height"] + pdf_width, pdf_height = pdf_dimensions[page_num] + + transformed_entry_box = transform_coordinates( + field["entry_bounding_box"], + image_width, image_height, + pdf_width, pdf_height + ) + + # Skip empty fields + if "entry_text" not in field or "text" not in field["entry_text"]: + continue + entry_text = field["entry_text"] + text = entry_text["text"] + if not text: + continue + + font_name = entry_text.get("font", "Arial") + font_size = str(entry_text.get("font_size", 14)) + "pt" + font_color = entry_text.get("font_color", "000000") + + # Font size/color seems to not work reliably across viewers: + # https://github.com/py-pdf/pypdf/issues/2084 + annotation = FreeText( + text=text, + rect=transformed_entry_box, + font=font_name, + font_size=font_size, + font_color=font_color, + border_color=None, + background_color=None, + ) + annotations.append(annotation) + # page_number is 0-based for pypdf + writer.add_annotation(page_number=page_num - 1, annotation=annotation) + + # Save the filled PDF + with open(output_pdf_path, "wb") as output: + writer.write(output) + + print(f"Successfully filled PDF form and saved to {output_pdf_path}") + print(f"Added {len(annotations)} text annotations") + + +if __name__ == "__main__": + if len(sys.argv) != 4: + print("Usage: fill_pdf_form_with_annotations.py [input pdf] [fields.json] [output pdf]") + sys.exit(1) + input_pdf = sys.argv[1] + fields_json = sys.argv[2] + output_pdf = sys.argv[3] + + fill_pdf_form(input_pdf, fields_json, output_pdf) \ No newline at end of file diff --git a/skills/document-skills/pptx/LICENSE.txt b/skills/document-skills/pptx/LICENSE.txt new file mode 100644 index 0000000..c55ab42 --- /dev/null +++ b/skills/document-skills/pptx/LICENSE.txt @@ -0,0 +1,30 @@ +© 2025 Anthropic, PBC. All rights reserved. + +LICENSE: Use of these materials (including all code, prompts, assets, files, +and other components of this Skill) is governed by your agreement with +Anthropic regarding use of Anthropic's services. If no separate agreement +exists, use is governed by Anthropic's Consumer Terms of Service or +Commercial Terms of Service, as applicable: +https://www.anthropic.com/legal/consumer-terms +https://www.anthropic.com/legal/commercial-terms +Your applicable agreement is referred to as the "Agreement." "Services" are +as defined in the Agreement. + +ADDITIONAL RESTRICTIONS: Notwithstanding anything in the Agreement to the +contrary, users may not: + +- Extract these materials from the Services or retain copies of these + materials outside the Services +- Reproduce or copy these materials, except for temporary copies created + automatically during authorized use of the Services +- Create derivative works based on these materials +- Distribute, sublicense, or transfer these materials to any third party +- Make, offer to sell, sell, or import any inventions embodied in these + materials +- Reverse engineer, decompile, or disassemble these materials + +The receipt, viewing, or possession of these materials does not convey or +imply any license or right beyond those expressly granted above. + +Anthropic retains all right, title, and interest in these materials, +including all copyrights, patents, and other intellectual property rights. diff --git a/skills/document-skills/pptx/SKILL.md b/skills/document-skills/pptx/SKILL.md new file mode 100644 index 0000000..b93b875 --- /dev/null +++ b/skills/document-skills/pptx/SKILL.md @@ -0,0 +1,484 @@ +--- +name: pptx +description: "Presentation creation, editing, and analysis. When Claude needs to work with presentations (.pptx files) for: (1) Creating new presentations, (2) Modifying or editing content, (3) Working with layouts, (4) Adding comments or speaker notes, or any other presentation tasks" +license: Proprietary. LICENSE.txt has complete terms +--- + +# PPTX creation, editing, and analysis + +## Overview + +A user may ask you to create, edit, or analyze the contents of a .pptx file. A .pptx file is essentially a ZIP archive containing XML files and other resources that you can read or edit. You have different tools and workflows available for different tasks. + +## Reading and analyzing content + +### Text extraction +If you just need to read the text contents of a presentation, you should convert the document to markdown: + +```bash +# Convert document to markdown +python -m markitdown path-to-file.pptx +``` + +### Raw XML access +You need raw XML access for: comments, speaker notes, slide layouts, animations, design elements, and complex formatting. For any of these features, you'll need to unpack a presentation and read its raw XML contents. + +#### Unpacking a file +`python ooxml/scripts/unpack.py ` + +**Note**: The unpack.py script is located at `skills/pptx/ooxml/scripts/unpack.py` relative to the project root. If the script doesn't exist at this path, use `find . -name "unpack.py"` to locate it. + +#### Key file structures +* `ppt/presentation.xml` - Main presentation metadata and slide references +* `ppt/slides/slide{N}.xml` - Individual slide contents (slide1.xml, slide2.xml, etc.) +* `ppt/notesSlides/notesSlide{N}.xml` - Speaker notes for each slide +* `ppt/comments/modernComment_*.xml` - Comments for specific slides +* `ppt/slideLayouts/` - Layout templates for slides +* `ppt/slideMasters/` - Master slide templates +* `ppt/theme/` - Theme and styling information +* `ppt/media/` - Images and other media files + +#### Typography and color extraction +**When given an example design to emulate**: Always analyze the presentation's typography and colors first using the methods below: +1. **Read theme file**: Check `ppt/theme/theme1.xml` for colors (``) and fonts (``) +2. **Sample slide content**: Examine `ppt/slides/slide1.xml` for actual font usage (``) and colors +3. **Search for patterns**: Use grep to find color (``, ``) and font references across all XML files + +## Creating a new PowerPoint presentation **without a template** + +When creating a new PowerPoint presentation from scratch, use the **html2pptx** workflow to convert HTML slides to PowerPoint with accurate positioning. + +### Design Principles + +**CRITICAL**: Before creating any presentation, analyze the content and choose appropriate design elements: +1. **Consider the subject matter**: What is this presentation about? What tone, industry, or mood does it suggest? +2. **Check for branding**: If the user mentions a company/organization, consider their brand colors and identity +3. **Match palette to content**: Select colors that reflect the subject +4. **State your approach**: Explain your design choices before writing code + +**Requirements**: +- ✅ State your content-informed design approach BEFORE writing code +- ✅ Use web-safe fonts only: Arial, Helvetica, Times New Roman, Georgia, Courier New, Verdana, Tahoma, Trebuchet MS, Impact +- ✅ Create clear visual hierarchy through size, weight, and color +- ✅ Ensure readability: strong contrast, appropriately sized text, clean alignment +- ✅ Be consistent: repeat patterns, spacing, and visual language across slides + +#### Color Palette Selection + +**Choosing colors creatively**: +- **Think beyond defaults**: What colors genuinely match this specific topic? Avoid autopilot choices. +- **Consider multiple angles**: Topic, industry, mood, energy level, target audience, brand identity (if mentioned) +- **Be adventurous**: Try unexpected combinations - a healthcare presentation doesn't have to be green, finance doesn't have to be navy +- **Build your palette**: Pick 3-5 colors that work together (dominant colors + supporting tones + accent) +- **Ensure contrast**: Text must be clearly readable on backgrounds + +**Example color palettes** (use these to spark creativity - choose one, adapt it, or create your own): + +1. **Classic Blue**: Deep navy (#1C2833), slate gray (#2E4053), silver (#AAB7B8), off-white (#F4F6F6) +2. **Teal & Coral**: Teal (#5EA8A7), deep teal (#277884), coral (#FE4447), white (#FFFFFF) +3. **Bold Red**: Red (#C0392B), bright red (#E74C3C), orange (#F39C12), yellow (#F1C40F), green (#2ECC71) +4. **Warm Blush**: Mauve (#A49393), blush (#EED6D3), rose (#E8B4B8), cream (#FAF7F2) +5. **Burgundy Luxury**: Burgundy (#5D1D2E), crimson (#951233), rust (#C15937), gold (#997929) +6. **Deep Purple & Emerald**: Purple (#B165FB), dark blue (#181B24), emerald (#40695B), white (#FFFFFF) +7. **Cream & Forest Green**: Cream (#FFE1C7), forest green (#40695B), white (#FCFCFC) +8. **Pink & Purple**: Pink (#F8275B), coral (#FF574A), rose (#FF737D), purple (#3D2F68) +9. **Lime & Plum**: Lime (#C5DE82), plum (#7C3A5F), coral (#FD8C6E), blue-gray (#98ACB5) +10. **Black & Gold**: Gold (#BF9A4A), black (#000000), cream (#F4F6F6) +11. **Sage & Terracotta**: Sage (#87A96B), terracotta (#E07A5F), cream (#F4F1DE), charcoal (#2C2C2C) +12. **Charcoal & Red**: Charcoal (#292929), red (#E33737), light gray (#CCCBCB) +13. **Vibrant Orange**: Orange (#F96D00), light gray (#F2F2F2), charcoal (#222831) +14. **Forest Green**: Black (#191A19), green (#4E9F3D), dark green (#1E5128), white (#FFFFFF) +15. **Retro Rainbow**: Purple (#722880), pink (#D72D51), orange (#EB5C18), amber (#F08800), gold (#DEB600) +16. **Vintage Earthy**: Mustard (#E3B448), sage (#CBD18F), forest green (#3A6B35), cream (#F4F1DE) +17. **Coastal Rose**: Old rose (#AD7670), beaver (#B49886), eggshell (#F3ECDC), ash gray (#BFD5BE) +18. **Orange & Turquoise**: Light orange (#FC993E), grayish turquoise (#667C6F), white (#FCFCFC) + +#### Visual Details Options + +**Geometric Patterns**: +- Diagonal section dividers instead of horizontal +- Asymmetric column widths (30/70, 40/60, 25/75) +- Rotated text headers at 90° or 270° +- Circular/hexagonal frames for images +- Triangular accent shapes in corners +- Overlapping shapes for depth + +**Border & Frame Treatments**: +- Thick single-color borders (10-20pt) on one side only +- Double-line borders with contrasting colors +- Corner brackets instead of full frames +- L-shaped borders (top+left or bottom+right) +- Underline accents beneath headers (3-5pt thick) + +**Typography Treatments**: +- Extreme size contrast (72pt headlines vs 11pt body) +- All-caps headers with wide letter spacing +- Numbered sections in oversized display type +- Monospace (Courier New) for data/stats/technical content +- Condensed fonts (Arial Narrow) for dense information +- Outlined text for emphasis + +**Chart & Data Styling**: +- Monochrome charts with single accent color for key data +- Horizontal bar charts instead of vertical +- Dot plots instead of bar charts +- Minimal gridlines or none at all +- Data labels directly on elements (no legends) +- Oversized numbers for key metrics + +**Layout Innovations**: +- Full-bleed images with text overlays +- Sidebar column (20-30% width) for navigation/context +- Modular grid systems (3×3, 4×4 blocks) +- Z-pattern or F-pattern content flow +- Floating text boxes over colored shapes +- Magazine-style multi-column layouts + +**Background Treatments**: +- Solid color blocks occupying 40-60% of slide +- Gradient fills (vertical or diagonal only) +- Split backgrounds (two colors, diagonal or vertical) +- Edge-to-edge color bands +- Negative space as a design element + +### Layout Tips +**When creating slides with charts or tables:** +- **Two-column layout (PREFERRED)**: Use a header spanning the full width, then two columns below - text/bullets in one column and the featured content in the other. This provides better balance and makes charts/tables more readable. Use flexbox with unequal column widths (e.g., 40%/60% split) to optimize space for each content type. +- **Full-slide layout**: Let the featured content (chart/table) take up the entire slide for maximum impact and readability +- **NEVER vertically stack**: Do not place charts/tables below text in a single column - this causes poor readability and layout issues + +### Workflow +1. **MANDATORY - READ ENTIRE FILE**: Read [`html2pptx.md`](html2pptx.md) completely from start to finish. **NEVER set any range limits when reading this file.** Read the full file content for detailed syntax, critical formatting rules, and best practices before proceeding with presentation creation. +2. Create an HTML file for each slide with proper dimensions (e.g., 720pt × 405pt for 16:9) + - Use `

`, `

`-`

`, `