283 lines
9.3 KiB
Python
283 lines
9.3 KiB
Python
"""
|
|
Export server for processing Google Sheets export jobs.
|
|
|
|
This module handles the actual processing of export jobs queued through
|
|
the FastStream message broker, including data retrieval, formatting,
|
|
and Google Sheets API integration.
|
|
"""
|
|
|
|
import logging
|
|
from typing import Dict, Any, List, Optional
|
|
from datetime import datetime
|
|
from adapters.sqlite.database import get_db
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
class ExportServer:
|
|
"""Server for processing Google Sheets export operations."""
|
|
|
|
def __init__(self):
|
|
"""Initialize ExportServer with database and Google Sheets client."""
|
|
self.db = get_db
|
|
# TODO: Initialize Google Sheets client when implemented
|
|
# self.sheets_client = GoogleSheetsClient()
|
|
|
|
async def process_export_job(self, job_data: Dict[str, Any]) -> None:
|
|
"""
|
|
Process a Google Sheets export job.
|
|
|
|
Args:
|
|
job_data: Dictionary containing job_id, configuration_id, and metadata
|
|
"""
|
|
job_id = job_data["job_id"]
|
|
configuration_id = job_data["configuration_id"]
|
|
|
|
try:
|
|
logger.info(f"Starting export job {job_id} for configuration {configuration_id}")
|
|
|
|
# Update job status to running
|
|
await self._update_job_status(job_id, "running", started_at=datetime.utcnow())
|
|
|
|
# Get export configuration
|
|
config = await self._get_export_configuration(configuration_id)
|
|
if not config:
|
|
raise ValueError(f"Export configuration {configuration_id} not found")
|
|
|
|
# Process each enabled entity
|
|
total_records = 0
|
|
processed_entities = {}
|
|
|
|
for entity_type, mapping in config["entity_mappings"].items():
|
|
if mapping.get("is_enabled", False):
|
|
logger.info(f"Exporting {entity_type} data")
|
|
|
|
records_count = await self._export_entity(
|
|
entity_type,
|
|
config["sheet_id"],
|
|
mapping,
|
|
config.get("date_range_start"),
|
|
config.get("date_range_end")
|
|
)
|
|
|
|
total_records += records_count
|
|
processed_entities[entity_type] = {
|
|
"processed": records_count,
|
|
"total": records_count,
|
|
"status": "completed"
|
|
}
|
|
|
|
logger.info(f"Exported {records_count} {entity_type} records")
|
|
|
|
# Update job as completed
|
|
await self._update_job_status(
|
|
job_id,
|
|
"completed",
|
|
completed_at=datetime.utcnow(),
|
|
records_processed=total_records,
|
|
total_records=total_records,
|
|
entities_processed=processed_entities
|
|
)
|
|
|
|
logger.info(f"Export job {job_id} completed successfully with {total_records} records")
|
|
|
|
except Exception as e:
|
|
logger.error(f"Export job {job_id} failed: {str(e)}")
|
|
await self._update_job_status(
|
|
job_id,
|
|
"failed",
|
|
completed_at=datetime.utcnow(),
|
|
error_message=str(e)
|
|
)
|
|
raise
|
|
|
|
async def _export_entity(
|
|
self,
|
|
entity_type: str,
|
|
sheet_id: str,
|
|
mapping: Dict[str, Any],
|
|
date_range_start: Optional[str] = None,
|
|
date_range_end: Optional[str] = None
|
|
) -> int:
|
|
"""
|
|
Export specific entity type to Google Sheets.
|
|
|
|
Args:
|
|
entity_type: Type of entity to export (deals, contacts, companies, etc.)
|
|
sheet_id: Google Sheets document ID
|
|
mapping: Field mapping configuration for this entity
|
|
date_range_start: Optional start date filter
|
|
date_range_end: Optional end date filter
|
|
|
|
Returns:
|
|
Number of records exported
|
|
"""
|
|
try:
|
|
# Get entity data from database
|
|
data = await self._get_entity_data(entity_type, date_range_start, date_range_end)
|
|
|
|
if not data:
|
|
logger.warning(f"No data found for {entity_type}")
|
|
return 0
|
|
|
|
# Format data according to mapping
|
|
formatted_data = await self._format_data_for_export(data, mapping)
|
|
|
|
# Export to Google Sheets
|
|
await self._write_to_google_sheets(sheet_id, mapping["sheet_name"], formatted_data)
|
|
|
|
return len(data)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Failed to export {entity_type}: {str(e)}")
|
|
raise
|
|
|
|
async def _get_entity_data(
|
|
self,
|
|
entity_type: str,
|
|
date_range_start: Optional[str] = None,
|
|
date_range_end: Optional[str] = None
|
|
) -> List[Dict[str, Any]]:
|
|
"""
|
|
Retrieve entity data from the database.
|
|
|
|
Args:
|
|
entity_type: Type of entity to retrieve
|
|
date_range_start: Optional start date filter
|
|
date_range_end: Optional end date filter
|
|
|
|
Returns:
|
|
List of entity records
|
|
"""
|
|
# TODO: Implement database query based on entity type and date range
|
|
# This would query the appropriate table (amo_deals, amo_contacts, etc.)
|
|
# and join with custom fields if needed
|
|
|
|
logger.info(f"Retrieving {entity_type} data (date range: {date_range_start} to {date_range_end})")
|
|
|
|
# Placeholder implementation
|
|
return []
|
|
|
|
async def _format_data_for_export(
|
|
self,
|
|
data: List[Dict[str, Any]],
|
|
mapping: Dict[str, Any]
|
|
) -> List[List[Any]]:
|
|
"""
|
|
Format data according to the export mapping configuration.
|
|
|
|
Args:
|
|
data: Raw entity data from database
|
|
mapping: Field mapping configuration
|
|
|
|
Returns:
|
|
Formatted data ready for Google Sheets export
|
|
"""
|
|
field_mapping = mapping.get("field_mapping", [])
|
|
|
|
# Sort field mapping by order
|
|
sorted_fields = sorted(field_mapping, key=lambda x: x.get("order", 0))
|
|
|
|
# Create header row
|
|
headers = [field["field_name"] for field in sorted_fields]
|
|
formatted_data = [headers]
|
|
|
|
# Format data rows
|
|
for record in data:
|
|
row = []
|
|
for field in sorted_fields:
|
|
field_name = field["field_name"]
|
|
value = record.get(field_name, "")
|
|
|
|
# Handle different data types and formatting
|
|
if isinstance(value, datetime):
|
|
value = value.isoformat()
|
|
elif value is None:
|
|
value = ""
|
|
|
|
row.append(value)
|
|
|
|
formatted_data.append(row)
|
|
|
|
logger.info(f"Formatted {len(data)} records with {len(headers)} columns")
|
|
return formatted_data
|
|
|
|
async def _write_to_google_sheets(
|
|
self,
|
|
sheet_id: str,
|
|
sheet_name: str,
|
|
data: List[List[Any]]
|
|
) -> None:
|
|
"""
|
|
Write formatted data to Google Sheets.
|
|
|
|
Args:
|
|
sheet_id: Google Sheets document ID
|
|
sheet_name: Name of the sheet tab
|
|
data: Formatted data to write
|
|
"""
|
|
# TODO: Implement Google Sheets API integration
|
|
# This would use the Google Sheets client to write data
|
|
|
|
logger.info(f"Writing {len(data)} rows to sheet '{sheet_name}' in document {sheet_id}")
|
|
|
|
# Placeholder - would actually write to Google Sheets
|
|
logger.info(f"Successfully wrote data to Google Sheets (placeholder)")
|
|
|
|
async def _get_export_configuration(self, configuration_id: int) -> Optional[Dict[str, Any]]:
|
|
"""
|
|
Get export configuration from database.
|
|
|
|
Args:
|
|
configuration_id: ID of the export configuration
|
|
|
|
Returns:
|
|
Export configuration data or None if not found
|
|
"""
|
|
# TODO: Implement database query to get export configuration
|
|
# This would query the export_configuration and export_entity_mappings tables
|
|
|
|
logger.info(f"Retrieving export configuration {configuration_id}")
|
|
|
|
# Placeholder implementation
|
|
return {
|
|
"id": configuration_id,
|
|
"sheet_id": "placeholder_sheet_id",
|
|
"entity_mappings": {
|
|
"deals": {
|
|
"sheet_name": "Deals",
|
|
"is_enabled": True,
|
|
"field_mapping": [
|
|
{"field_name": "name", "column": "A", "order": 1},
|
|
{"field_name": "price", "column": "B", "order": 2},
|
|
{"field_name": "created_at", "column": "C", "order": 3}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
|
|
async def _update_job_status(
|
|
self,
|
|
job_id: str,
|
|
status: str,
|
|
**kwargs
|
|
) -> None:
|
|
"""
|
|
Update job status in the database.
|
|
|
|
Args:
|
|
job_id: Unique identifier of the job
|
|
status: New status (pending, running, completed, failed)
|
|
**kwargs: Additional fields to update
|
|
"""
|
|
# TODO: Implement database update
|
|
# This would update the export_jobs table
|
|
|
|
logger.info(f"Updating job {job_id} status to {status}")
|
|
|
|
update_data = {"status": status, **kwargs}
|
|
logger.debug(f"Job update data: {update_data}")
|
|
|
|
# Placeholder - would actually update database
|
|
pass
|