personal-projects/personal-portfolio
Template
1
1
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

fix: Use venv dbt and show full error output

Browse Source 
- Use .venv/bin/dbt if available, fall back to system dbt
- Show both stdout and stderr on dbt failures for better debugging

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
Leo Miranda
2026-01-18 12:18:26 -05:00
parent 2fc2a1bdb5
commit ebe48304d7
1 changed files with 10 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
scripts/data/load_toronto_data.py
View File

@@ -288,6 +288,10 @@ class DataPipeline:
logger.info("Running dbt transformations...") logger.info("Running dbt transformations...")
dbt_project_dir = PROJECT_ROOT / "dbt" dbt_project_dir = PROJECT_ROOT / "dbt"
venv_dbt = PROJECT_ROOT / ".venv" / "bin" / "dbt"
# Use venv dbt if available, otherwise fall back to system dbt
dbt_cmd = str(venv_dbt) if venv_dbt.exists() else "dbt"
if not dbt_project_dir.exists(): if not dbt_project_dir.exists():
logger.error(f"dbt project directory not found: {dbt_project_dir}") logger.error(f"dbt project directory not found: {dbt_project_dir}")
@@ -302,16 +306,14 @@ class DataPipeline:
# Run dbt models # Run dbt models
logger.info(" Running dbt run...") logger.info(" Running dbt run...")
result = subprocess.run( result = subprocess.run(
["dbt", "run", "--profiles-dir", str(dbt_project_dir)], [dbt_cmd, "run", "--profiles-dir", str(dbt_project_dir)],
cwd=dbt_project_dir, cwd=dbt_project_dir,
capture_output=True, capture_output=True,
text=True, text=True,
) )
if result.returncode != 0: if result.returncode != 0:
logger.error(f"dbt run failed:\n{result.stderr}") logger.error(f"dbt run failed:\n{result.stdout}\n{result.stderr}")
if self.verbose:
logger.debug(f"dbt output:\n{result.stdout}")
return False return False
logger.info(" dbt run completed successfully") logger.info(" dbt run completed successfully")
@@ -319,14 +321,16 @@ class DataPipeline:
# Run dbt tests # Run dbt tests
logger.info(" Running dbt test...") logger.info(" Running dbt test...")
result = subprocess.run( result = subprocess.run(
["dbt", "test", "--profiles-dir", str(dbt_project_dir)], [dbt_cmd, "test", "--profiles-dir", str(dbt_project_dir)],
cwd=dbt_project_dir, cwd=dbt_project_dir,
capture_output=True, capture_output=True,
text=True, text=True,
) )
if result.returncode != 0: if result.returncode != 0:
logger.warning(f"dbt test had failures:\n{result.stderr}") logger.warning(
f"dbt test had failures:\n{result.stdout}\n{result.stderr}"
)
# Don't fail on test failures, just warn # Don't fail on test failures, just warn
else: else:
logger.info(" dbt test completed successfully") logger.info(" dbt test completed successfully")