9.9 KiB
9.9 KiB
Getting Started with Data Commons
Quick Start Guide
This guide provides end-to-end examples for common Data Commons workflows.
Installation and Setup
# Install with Pandas support
pip install "datacommons-client[Pandas]"
# Set up API key for datacommons.org
export DC_API_KEY="your_api_key_here"
Request an API key at: https://apikeys.datacommons.org/
Example 1: Basic Population Query
Query current population for specific places:
from datacommons_client import DataCommonsClient
# Initialize client
client = DataCommonsClient()
# Step 1: Resolve place names to DCIDs
places = ["California", "Texas", "New York"]
resolve_response = client.resolve.fetch_dcids_by_name(
names=places,
entity_type="State"
)
# Extract DCIDs
dcids = []
for name, result in resolve_response.to_dict().items():
if result["candidates"]:
dcids.append(result["candidates"][0]["dcid"])
print(f"{name}: {result['candidates'][0]['dcid']}")
# Step 2: Query population data
response = client.observation.fetch(
variable_dcids=["Count_Person"],
entity_dcids=dcids,
date="latest"
)
# Step 3: Display results
data = response.to_dict()
for variable, entities in data.items():
for entity, observations in entities.items():
for obs in observations:
print(f"{entity}: {obs['value']:,} people ({obs['date']})")
Example 2: Time Series Analysis
Retrieve and plot historical unemployment rates:
import pandas as pd
import matplotlib.pyplot as plt
client = DataCommonsClient()
# Query unemployment rate over time
response = client.observation.fetch(
variable_dcids=["UnemploymentRate_Person"],
entity_dcids=["country/USA"],
date="all" # Get all historical data
)
# Convert to DataFrame
df = response.to_observations_as_records()
# Plot
df = df.sort_values('date')
plt.figure(figsize=(12, 6))
plt.plot(df['date'], df['value'])
plt.title('US Unemployment Rate Over Time')
plt.xlabel('Year')
plt.ylabel('Unemployment Rate (%)')
plt.grid(True)
plt.show()
Example 3: Geographic Hierarchy Query
Get data for all counties within a state:
client = DataCommonsClient()
# Query median income for all California counties
response = client.observation.fetch(
variable_dcids=["Median_Income_Household"],
entity_expression="geoId/06<-containedInPlace+{typeOf:County}",
date="2020"
)
# Convert to DataFrame and sort
df = response.to_observations_as_records()
# Get county names
county_dcids = df['entity'].unique().tolist()
names = client.node.fetch_entity_names(node_dcids=county_dcids)
# Add names to dataframe
df['name'] = df['entity'].map(names)
# Display top 10 by income
top_counties = df.nlargest(10, 'value')[['name', 'value']]
print("\nTop 10 California Counties by Median Household Income:")
for idx, row in top_counties.iterrows():
print(f"{row['name']}: ${row['value']:,.0f}")
Example 4: Multi-Variable Comparison
Compare multiple statistics across entities:
import pandas as pd
client = DataCommonsClient()
# Define places
places = ["California", "Texas", "Florida", "New York"]
resolve_response = client.resolve.fetch_dcids_by_name(names=places)
dcids = []
name_map = {}
for name, result in resolve_response.to_dict().items():
if result["candidates"]:
dcid = result["candidates"][0]["dcid"]
dcids.append(dcid)
name_map[dcid] = name
# Query multiple variables
variables = [
"Count_Person",
"Median_Income_Household",
"UnemploymentRate_Person",
"Median_Age_Person"
]
response = client.observation.fetch(
variable_dcids=variables,
entity_dcids=dcids,
date="latest"
)
# Convert to DataFrame
df = response.to_observations_as_records()
# Add readable names
df['state'] = df['entity'].map(name_map)
# Pivot for comparison
pivot = df.pivot_table(
values='value',
index='state',
columns='variable'
)
print("\nState Comparison:")
print(pivot.to_string())
Example 5: Coordinate-Based Query
Find and query data for a location by coordinates:
client = DataCommonsClient()
# User provides coordinates (e.g., from GPS)
latitude, longitude = 37.7749, -122.4194 # San Francisco
# Step 1: Resolve coordinates to place
dcid = client.resolve.fetch_dcid_by_coordinates(
latitude=latitude,
longitude=longitude
)
# Step 2: Get place name
name = client.node.fetch_entity_names(node_dcids=[dcid])
print(f"Location: {name[dcid]}")
# Step 3: Check available variables
available_vars = client.observation.fetch_available_statistical_variables(
entity_dcids=[dcid]
)
print(f"\nAvailable variables: {len(available_vars[dcid])} found")
print("First 10:", list(available_vars[dcid])[:10])
# Step 4: Query specific variables
response = client.observation.fetch(
variable_dcids=["Count_Person", "Median_Income_Household"],
entity_dcids=[dcid],
date="latest"
)
# Display results
df = response.to_observations_as_records()
print("\nStatistics:")
for _, row in df.iterrows():
print(f"{row['variable']}: {row['value']}")
Example 6: Data Source Filtering
Query data from specific sources for consistency:
client = DataCommonsClient()
# Query with facet filtering
response = client.observation.fetch(
variable_dcids=["Count_Person"],
entity_dcids=["country/USA"],
date="all",
filter_facet_domains=["census.gov"] # Only US Census data
)
df = response.to_observations_as_records()
print(f"Found {len(df)} observations from census.gov")
# Compare with all sources
response_all = client.observation.fetch(
variable_dcids=["Count_Person"],
entity_dcids=["country/USA"],
date="all"
)
df_all = response_all.to_observations_as_records()
print(f"Found {len(df_all)} observations from all sources")
Example 7: Exploring the Knowledge Graph
Discover entity properties and relationships:
client = DataCommonsClient()
# Step 1: Explore what properties exist
entity = "geoId/06" # California
# Get outgoing properties
out_props = client.node.fetch_property_labels(
node_dcids=[entity],
out=True
)
print(f"Outgoing properties for California:")
print(out_props[entity])
# Get incoming properties
in_props = client.node.fetch_property_labels(
node_dcids=[entity],
out=False
)
print(f"\nIncoming properties for California:")
print(in_props[entity])
# Step 2: Get specific property values
name_response = client.node.fetch_property_values(
node_dcids=[entity],
property="name",
out=True
)
print(f"\nName property value:")
print(name_response.to_dict())
# Step 3: Explore hierarchy
children = client.node.fetch_place_children(node_dcids=[entity])
print(f"\nNumber of child places: {len(children[entity])}")
# Get names for first 5 children
if children[entity]:
child_sample = children[entity][:5]
child_names = client.node.fetch_entity_names(node_dcids=child_sample)
print("\nSample child places:")
for dcid, name in child_names.items():
print(f" {name}")
Example 8: Batch Processing Multiple Queries
Efficiently query data for many entities:
import pandas as pd
client = DataCommonsClient()
# List of cities to analyze
cities = [
"San Francisco, CA",
"Los Angeles, CA",
"San Diego, CA",
"Sacramento, CA",
"San Jose, CA"
]
# Resolve all cities
resolve_response = client.resolve.fetch_dcids_by_name(
names=cities,
entity_type="City"
)
# Build mapping
city_dcids = []
dcid_to_name = {}
for name, result in resolve_response.to_dict().items():
if result["candidates"]:
dcid = result["candidates"][0]["dcid"]
city_dcids.append(dcid)
dcid_to_name[dcid] = name
# Query multiple variables at once
variables = [
"Count_Person",
"Median_Income_Household",
"UnemploymentRate_Person"
]
response = client.observation.fetch(
variable_dcids=variables,
entity_dcids=city_dcids,
date="latest"
)
# Process into a comparison table
df = response.to_observations_as_records()
df['city'] = df['entity'].map(dcid_to_name)
# Create comparison table
comparison = df.pivot_table(
values='value',
index='city',
columns='variable',
aggfunc='first'
)
print("\nCalifornia Cities Comparison:")
print(comparison.to_string())
# Export to CSV
comparison.to_csv('ca_cities_comparison.csv')
print("\nData exported to ca_cities_comparison.csv")
Common Patterns Summary
Pattern 1: Name → DCID → Data
names = ["California"]
dcids = resolve_names(names)
data = query_observations(dcids, variables)
Pattern 2: Coordinates → DCID → Data
dcid = resolve_coordinates(lat, lon)
data = query_observations([dcid], variables)
Pattern 3: Parent → Children → Data
children = get_place_children(parent_dcid)
data = query_observations(children, variables)
Pattern 4: Explore → Select → Query
available_vars = check_available_variables(dcids)
selected_vars = filter_relevant(available_vars)
data = query_observations(dcids, selected_vars)
Error Handling Best Practices
client = DataCommonsClient()
# Always check for candidates
resolve_response = client.resolve.fetch_dcids_by_name(names=["Unknown Place"])
result = resolve_response.to_dict()["Unknown Place"]
if not result["candidates"]:
print("No matches found - try a more specific name")
# Handle error appropriately
else:
dcid = result["candidates"][0]["dcid"]
# Proceed with query
# Check for multiple candidates (ambiguity)
if len(result["candidates"]) > 1:
print(f"Multiple matches found: {len(result['candidates'])}")
for candidate in result["candidates"]:
print(f" {candidate['dcid']} ({candidate.get('dominantType', 'N/A')})")
# Let user select or use additional filtering
Next Steps
- Explore available statistical variables: https://datacommons.org/tools/statvar
- Browse the knowledge graph: https://datacommons.org/browser/
- Read detailed endpoint documentation in
references/directory - Check official documentation: https://docs.datacommons.org/api/python/v2/