34 lines
5.7 KiB
JSON
34 lines
5.7 KiB
JSON
{
|
|
"name": "Data Analytics Pipeline",
|
|
"description": "ETL, analysis, and visualization with Docker (MariaDB + Jupyter)",
|
|
"keywords": ["data", "analytics", "csv", "etl", "visualization", "statistics", "dashboard", "jupyter", "pandas", "matplotlib"],
|
|
"files": {
|
|
"etl.py": {
|
|
"description": "Data loading, cleaning, and transformation",
|
|
"example": "import pandas as pd\nfrom pathlib import Path\nfrom sqlalchemy import create_engine\n\nDB_URL = \"mysql+pymysql://root:secret@localhost:3306/analytics\"\nengine = create_engine(DB_URL)\n\ndef load_csv(path: str) -> pd.DataFrame:\n df = pd.read_csv(path)\n print(f\"Loaded {len(df)} rows from {path}\")\n return df\n\ndef clean(df: pd.DataFrame) -> pd.DataFrame:\n df = df.dropna(subset=[\"x\", \"y\"])\n df = df[(df[\"x\"] >= 0) & (df[\"y\"] >= 0)] # Remove outliers\n df[\"timestamp\"] = pd.to_datetime(df[\"timestamp\"])\n return df.sort_values(\"timestamp\").reset_index(drop=True)\n\ndef to_database(df: pd.DataFrame, table: str):\n df.to_sql(table, engine, if_exists=\"replace\", index=False)\n print(f\"Wrote {len(df)} rows to {table}\")\n\nif __name__ == \"__main__\":\n for csv_file in sorted(Path(\"data\").glob(\"*.csv\")):\n df = load_csv(str(csv_file))\n df = clean(df)\n to_database(df, \"measurements\")",
|
|
"instructions": "Write the ETL pipeline:\n- Load CSV files from data/ directory using pandas\n- Clean: remove nulls, filter outliers, parse timestamps\n- Transform: convert units, compute derived columns\n- Load into MariaDB via SQLAlchemy\n- Make it runnable as a standalone script"
|
|
},
|
|
"analysis.py": {
|
|
"description": "Statistical analysis and metrics computation",
|
|
"example": "import pandas as pd\nfrom sqlalchemy import create_engine\n\nDB_URL = \"mysql+pymysql://root:secret@localhost:3306/analytics\"\nengine = create_engine(DB_URL)\n\ndef load_data() -> pd.DataFrame:\n return pd.read_sql(\"SELECT * FROM measurements\", engine)\n\ndef summary_stats(df: pd.DataFrame) -> dict:\n return {\n \"total_rows\": len(df),\n \"date_range\": f\"{df['timestamp'].min()} to {df['timestamp'].max()}\",\n \"unique_entities\": df[\"entity_id\"].nunique(),\n }\n\ndef hourly_distribution(df: pd.DataFrame) -> pd.DataFrame:\n df[\"hour\"] = df[\"timestamp\"].dt.hour\n return df.groupby(\"hour\").size().reset_index(name=\"count\")\n\nif __name__ == \"__main__\":\n df = load_data()\n stats = summary_stats(df)\n for k, v in stats.items():\n print(f\"{k}: {v}\")",
|
|
"instructions": "Write analysis functions:\n- Load cleaned data from MariaDB\n- Compute summary statistics (counts, date ranges, distributions)\n- Time-based analysis (hourly, daily, weekly patterns)\n- Group-level metrics (per entity, per zone)\n- Return DataFrames and dicts suitable for visualization"
|
|
},
|
|
"visualize.py": {
|
|
"description": "Charts and visualizations with matplotlib",
|
|
"example": "import matplotlib.pyplot as plt\nimport pandas as pd\nfrom analysis import load_data, hourly_distribution\n\ndef plot_heatmap(df: pd.DataFrame, title: str, output: str):\n fig, ax = plt.subplots(figsize=(12, 8))\n scatter = ax.scatter(df[\"x\"], df[\"y\"], c=df[\"density\"], cmap=\"hot\", alpha=0.5, s=2)\n ax.set_title(title)\n ax.set_xlabel(\"x\")\n ax.set_ylabel(\"y\")\n ax.invert_yaxis()\n plt.colorbar(scatter, label=\"Density\")\n plt.tight_layout()\n plt.savefig(output, dpi=150)\n print(f\"Saved {output}\")\n\ndef plot_bar(df: pd.DataFrame, x: str, y: str, title: str, output: str):\n fig, ax = plt.subplots(figsize=(10, 5))\n ax.bar(df[x], df[y], color=\"steelblue\")\n ax.set_title(title)\n ax.set_xlabel(x)\n ax.set_ylabel(y)\n plt.tight_layout()\n plt.savefig(output, dpi=150)\n\nif __name__ == \"__main__\":\n df = load_data()\n hourly = hourly_distribution(df)\n plot_bar(hourly, \"hour\", \"count\", \"Hourly Distribution\", \"output/hourly.png\")",
|
|
"instructions": "Write visualization functions:\n- Import analysis functions for data\n- Heatmaps, bar charts, line charts as appropriate\n- Save figures to output/ directory (PNG, 150 DPI)\n- Use matplotlib with clear titles, labels, colorbars\n- Make it runnable as standalone to generate all charts"
|
|
},
|
|
"docker-compose.yml": {
|
|
"description": "Docker Compose stack for database and Jupyter",
|
|
"example": "services:\n db:\n image: mariadb:11\n environment:\n MYSQL_ROOT_PASSWORD: secret\n MYSQL_DATABASE: analytics\n ports:\n - \"3306:3306\"\n volumes:\n - db_data:/var/lib/mysql\n\n jupyter:\n image: jupyter/scipy-notebook:latest\n ports:\n - \"8888:8888\"\n volumes:\n - .:/home/jovyan/work\n environment:\n JUPYTER_TOKEN: kipina\n depends_on:\n - db\n\nvolumes:\n db_data:",
|
|
"instructions": "Write docker-compose.yml:\n- MariaDB service with persistent volume\n- JupyterLab service with project mounted\n- Correct environment variables\n- Port mappings for local development\n- Write ONLY the YAML, no explanations"
|
|
},
|
|
"pyproject.toml": {
|
|
"description": "Project dependencies",
|
|
"example": "[project]\nname = \"analytics\"\nversion = \"0.1.0\"\nrequires-python = \">=3.11\"\ndependencies = [\n \"pandas\",\n \"matplotlib\",\n \"sqlalchemy\",\n \"pymysql\",\n]\n\n[project.scripts]\netl = \"python etl.py\"\nanalyze = \"python analysis.py\"\nvisualize = \"python visualize.py\"",
|
|
"instructions": "Use [project] format (PEP 621). List all data science dependencies. Add scripts for ETL, analysis, and visualization."
|
|
}
|
|
},
|
|
"order": ["etl.py", "analysis.py", "visualize.py", "docker-compose.yml", "pyproject.toml"]
|
|
}
|