Initial commit
This commit is contained in:
Zhongwei Li
298
skills/gene-database/scripts/batch_gene_lookup.py
Normal file
298
skills/gene-database/scripts/batch_gene_lookup.py
Normal file
@@ -0,0 +1,298 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Batch gene lookup using NCBI APIs.
|
||||
|
||||
This script efficiently processes multiple gene queries with proper
|
||||
rate limiting and error handling.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from typing import Optional, List, Dict, Any
|
||||
|
||||
|
||||
def read_gene_list(filepath: str) -> List[str]:
|
||||
"""
|
||||
Read gene identifiers from a file (one per line).
|
||||
|
||||
Args:
|
||||
filepath: Path to file containing gene symbols or IDs
|
||||
|
||||
Returns:
|
||||
List of gene identifiers
|
||||
"""
|
||||
try:
|
||||
with open(filepath, 'r') as f:
|
||||
genes = [line.strip() for line in f if line.strip()]
|
||||
return genes
|
||||
except FileNotFoundError:
|
||||
print(f"Error: File '{filepath}' not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
print(f"Error reading file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def batch_esearch(queries: List[str], organism: Optional[str] = None,
|
||||
api_key: Optional[str] = None) -> Dict[str, str]:
|
||||
"""
|
||||
Search for multiple gene symbols and return their IDs.
|
||||
|
||||
Args:
|
||||
queries: List of gene symbols
|
||||
organism: Optional organism filter
|
||||
api_key: Optional NCBI API key
|
||||
|
||||
Returns:
|
||||
Dictionary mapping gene symbol to Gene ID (or 'NOT_FOUND')
|
||||
"""
|
||||
base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
|
||||
results = {}
|
||||
|
||||
# Rate limiting
|
||||
delay = 0.1 if api_key else 0.34 # 10 req/sec with key, 3 req/sec without
|
||||
|
||||
for query in queries:
|
||||
# Build search term
|
||||
search_term = f"{query}[gene]"
|
||||
if organism:
|
||||
search_term += f" AND {organism}[organism]"
|
||||
|
||||
params = {
|
||||
'db': 'gene',
|
||||
'term': search_term,
|
||||
'retmax': 1,
|
||||
'retmode': 'json'
|
||||
}
|
||||
|
||||
if api_key:
|
||||
params['api_key'] = api_key
|
||||
|
||||
url = f"{base_url}esearch.fcgi?{urllib.parse.urlencode(params)}"
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
if 'esearchresult' in data and 'idlist' in data['esearchresult']:
|
||||
id_list = data['esearchresult']['idlist']
|
||||
results[query] = id_list[0] if id_list else 'NOT_FOUND'
|
||||
else:
|
||||
results[query] = 'ERROR'
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error searching for {query}: {e}", file=sys.stderr)
|
||||
results[query] = 'ERROR'
|
||||
|
||||
time.sleep(delay)
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def batch_esummary(gene_ids: List[str], api_key: Optional[str] = None,
|
||||
chunk_size: int = 200) -> Dict[str, Dict[str, Any]]:
|
||||
"""
|
||||
Get summaries for multiple genes in batches.
|
||||
|
||||
Args:
|
||||
gene_ids: List of Gene IDs
|
||||
api_key: Optional NCBI API key
|
||||
chunk_size: Number of IDs per request (max 500)
|
||||
|
||||
Returns:
|
||||
Dictionary mapping Gene ID to summary data
|
||||
"""
|
||||
base_url = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
|
||||
all_results = {}
|
||||
|
||||
# Rate limiting
|
||||
delay = 0.1 if api_key else 0.34
|
||||
|
||||
# Process in chunks
|
||||
for i in range(0, len(gene_ids), chunk_size):
|
||||
chunk = gene_ids[i:i + chunk_size]
|
||||
|
||||
params = {
|
||||
'db': 'gene',
|
||||
'id': ','.join(chunk),
|
||||
'retmode': 'json'
|
||||
}
|
||||
|
||||
if api_key:
|
||||
params['api_key'] = api_key
|
||||
|
||||
url = f"{base_url}esummary.fcgi?{urllib.parse.urlencode(params)}"
|
||||
|
||||
try:
|
||||
with urllib.request.urlopen(url) as response:
|
||||
data = json.loads(response.read().decode())
|
||||
|
||||
if 'result' in data:
|
||||
for gene_id in chunk:
|
||||
if gene_id in data['result']:
|
||||
all_results[gene_id] = data['result'][gene_id]
|
||||
|
||||
except Exception as e:
|
||||
print(f"Error fetching summaries for chunk: {e}", file=sys.stderr)
|
||||
|
||||
time.sleep(delay)
|
||||
|
||||
return all_results
|
||||
|
||||
|
||||
def batch_lookup_by_ids(gene_ids: List[str], api_key: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Lookup genes by IDs and return structured data.
|
||||
|
||||
Args:
|
||||
gene_ids: List of Gene IDs
|
||||
api_key: Optional NCBI API key
|
||||
|
||||
Returns:
|
||||
List of gene information dictionaries
|
||||
"""
|
||||
summaries = batch_esummary(gene_ids, api_key=api_key)
|
||||
|
||||
results = []
|
||||
for gene_id in gene_ids:
|
||||
if gene_id in summaries:
|
||||
gene = summaries[gene_id]
|
||||
results.append({
|
||||
'gene_id': gene_id,
|
||||
'symbol': gene.get('name', 'N/A'),
|
||||
'description': gene.get('description', 'N/A'),
|
||||
'organism': gene.get('organism', {}).get('scientificname', 'N/A'),
|
||||
'chromosome': gene.get('chromosome', 'N/A'),
|
||||
'map_location': gene.get('maplocation', 'N/A'),
|
||||
'type': gene.get('geneticsource', 'N/A')
|
||||
})
|
||||
else:
|
||||
results.append({
|
||||
'gene_id': gene_id,
|
||||
'error': 'Not found or error fetching'
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def batch_lookup_by_symbols(gene_symbols: List[str], organism: str,
|
||||
api_key: Optional[str] = None) -> List[Dict[str, Any]]:
|
||||
"""
|
||||
Lookup genes by symbols and return structured data.
|
||||
|
||||
Args:
|
||||
gene_symbols: List of gene symbols
|
||||
organism: Organism name
|
||||
api_key: Optional NCBI API key
|
||||
|
||||
Returns:
|
||||
List of gene information dictionaries
|
||||
"""
|
||||
# First, search for IDs
|
||||
print(f"Searching for {len(gene_symbols)} gene symbols...", file=sys.stderr)
|
||||
symbol_to_id = batch_esearch(gene_symbols, organism=organism, api_key=api_key)
|
||||
|
||||
# Filter to valid IDs
|
||||
valid_ids = [id for id in symbol_to_id.values() if id not in ['NOT_FOUND', 'ERROR']]
|
||||
|
||||
if not valid_ids:
|
||||
print("No genes found", file=sys.stderr)
|
||||
return []
|
||||
|
||||
print(f"Found {len(valid_ids)} genes, fetching details...", file=sys.stderr)
|
||||
|
||||
# Fetch summaries
|
||||
summaries = batch_esummary(valid_ids, api_key=api_key)
|
||||
|
||||
# Build results
|
||||
results = []
|
||||
for symbol, gene_id in symbol_to_id.items():
|
||||
if gene_id == 'NOT_FOUND':
|
||||
results.append({
|
||||
'query_symbol': symbol,
|
||||
'status': 'not_found'
|
||||
})
|
||||
elif gene_id == 'ERROR':
|
||||
results.append({
|
||||
'query_symbol': symbol,
|
||||
'status': 'error'
|
||||
})
|
||||
elif gene_id in summaries:
|
||||
gene = summaries[gene_id]
|
||||
results.append({
|
||||
'query_symbol': symbol,
|
||||
'gene_id': gene_id,
|
||||
'symbol': gene.get('name', 'N/A'),
|
||||
'description': gene.get('description', 'N/A'),
|
||||
'organism': gene.get('organism', {}).get('scientificname', 'N/A'),
|
||||
'chromosome': gene.get('chromosome', 'N/A'),
|
||||
'map_location': gene.get('maplocation', 'N/A'),
|
||||
'type': gene.get('geneticsource', 'N/A')
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Batch gene lookup using NCBI APIs',
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Lookup by gene IDs
|
||||
%(prog)s --ids 672,7157,5594
|
||||
|
||||
# Lookup by symbols from a file
|
||||
%(prog)s --file genes.txt --organism human
|
||||
|
||||
# Lookup with API key and save to file
|
||||
%(prog)s --ids 672,7157,5594 --api-key YOUR_KEY --output results.json
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument('--ids', '-i', help='Comma-separated Gene IDs')
|
||||
parser.add_argument('--file', '-f', help='File containing gene symbols (one per line)')
|
||||
parser.add_argument('--organism', '-o', help='Organism name (required with --file)')
|
||||
parser.add_argument('--output', '-O', help='Output file path (JSON format)')
|
||||
parser.add_argument('--api-key', '-k', help='NCBI API key')
|
||||
parser.add_argument('--pretty', '-p', action='store_true',
|
||||
help='Pretty-print JSON output')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.ids and not args.file:
|
||||
parser.error("Either --ids or --file must be provided")
|
||||
|
||||
if args.file and not args.organism:
|
||||
parser.error("--organism is required when using --file")
|
||||
|
||||
# Process genes
|
||||
if args.ids:
|
||||
gene_ids = [id.strip() for id in args.ids.split(',')]
|
||||
results = batch_lookup_by_ids(gene_ids, api_key=args.api_key)
|
||||
else:
|
||||
gene_symbols = read_gene_list(args.file)
|
||||
results = batch_lookup_by_symbols(gene_symbols, args.organism, api_key=args.api_key)
|
||||
|
||||
# Output results
|
||||
indent = 2 if args.pretty else None
|
||||
json_output = json.dumps(results, indent=indent)
|
||||
|
||||
if args.output:
|
||||
try:
|
||||
with open(args.output, 'w') as f:
|
||||
f.write(json_output)
|
||||
print(f"Results written to {args.output}", file=sys.stderr)
|
||||
except Exception as e:
|
||||
print(f"Error writing output file: {e}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print(json_output)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user