Maxim Snesarev
33d6bb7ebd
- Updated database configuration to switch from SQLite to PostgreSQL, including changes to alembic.ini, Docker Compose, and environment settings. - Refactored application code to utilize PostgreSQL database adapters, ensuring compatibility with the new database structure. - Enhanced API routes and data handling to support the new database, including adjustments in data models and query logic. - Introduced new job processing mechanisms for full synchronization of AMO CRM entities, leveraging FastStream for background tasks. - Improved logging and error handling across the application to facilitate better monitoring and debugging. - Removed obsolete SQLite adapter files and migrations, streamlining the project structure for PostgreSQL integration.
123 lines
3.4 KiB
Python
123 lines
3.4 KiB
Python
"""
|
|
Task scheduler for periodic AMO CRM data refresh jobs.
|
|
|
|
This module sets up scheduled tasks using APScheduler
|
|
for periodic data synchronization and maintenance operations.
|
|
"""
|
|
|
|
import logging
|
|
import asyncio
|
|
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
|
from apscheduler.triggers.cron import CronTrigger
|
|
from workers.broker import broker
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Initialize APScheduler
|
|
scheduler = AsyncIOScheduler()
|
|
|
|
|
|
async def publish_refresh_job(entity_type: str):
|
|
"""Publish a refresh job to the broker."""
|
|
try:
|
|
logger.info(f"Publishing scheduled refresh job for {entity_type}")
|
|
await broker.publish(entity_type, channel="refresh-jobs")
|
|
logger.info(f"Successfully published refresh job for {entity_type}")
|
|
except Exception as e:
|
|
logger.error(f"Failed to publish refresh job for {entity_type}: {e}")
|
|
|
|
|
|
def schedule_refresh_jobs():
|
|
"""Schedule all periodic refresh jobs."""
|
|
|
|
# Deals: Every 6 hours
|
|
scheduler.add_job(
|
|
lambda: asyncio.create_task(publish_refresh_job("deals")),
|
|
CronTrigger(hour="*/6", minute="0"),
|
|
id="refresh_deals",
|
|
name="Refresh deals data",
|
|
replace_existing=True
|
|
)
|
|
|
|
# Contacts: Every 4 hours
|
|
scheduler.add_job(
|
|
lambda: asyncio.create_task(publish_refresh_job("contacts")),
|
|
CronTrigger(hour="*/4", minute="0"),
|
|
id="refresh_contacts",
|
|
name="Refresh contacts data",
|
|
replace_existing=True
|
|
)
|
|
|
|
# Companies: Every 8 hours
|
|
scheduler.add_job(
|
|
lambda: asyncio.create_task(publish_refresh_job("companies")),
|
|
CronTrigger(hour="*/8", minute="0"),
|
|
id="refresh_companies",
|
|
name="Refresh companies data",
|
|
replace_existing=True
|
|
)
|
|
|
|
# Users: Every 12 hours
|
|
scheduler.add_job(
|
|
lambda: asyncio.create_task(publish_refresh_job("users")),
|
|
CronTrigger(hour="*/12", minute="0"),
|
|
id="refresh_users",
|
|
name="Refresh users data",
|
|
replace_existing=True
|
|
)
|
|
|
|
# Pipelines: Daily
|
|
scheduler.add_job(
|
|
lambda: asyncio.create_task(publish_refresh_job("pipelines")),
|
|
CronTrigger(hour="0", minute="0"),
|
|
id="refresh_pipelines",
|
|
name="Refresh pipelines data",
|
|
replace_existing=True
|
|
)
|
|
|
|
# Events: Every 2 hours
|
|
scheduler.add_job(
|
|
lambda: asyncio.create_task(publish_refresh_job("events")),
|
|
CronTrigger(hour="*/2", minute="0"),
|
|
id="refresh_events",
|
|
name="Refresh events data",
|
|
replace_existing=True
|
|
)
|
|
|
|
logger.info("All refresh jobs scheduled successfully")
|
|
|
|
|
|
async def start_scheduler():
|
|
"""Start the task scheduler."""
|
|
logger.info("Starting APScheduler for periodic refresh jobs")
|
|
await broker.start()
|
|
schedule_refresh_jobs()
|
|
scheduler.start()
|
|
logger.info("Scheduler started successfully")
|
|
|
|
|
|
async def stop_scheduler():
|
|
"""Stop the task scheduler."""
|
|
logger.info("Stopping scheduler")
|
|
scheduler.shutdown(wait=True)
|
|
await broker.close()
|
|
logger.info("Scheduler stopped")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# This allows running the scheduler directly
|
|
import asyncio
|
|
|
|
async def main():
|
|
await start_scheduler()
|
|
try:
|
|
# Keep the scheduler running
|
|
while True:
|
|
await asyncio.sleep(60)
|
|
except KeyboardInterrupt:
|
|
logger.info("Scheduler interrupted by user")
|
|
finally:
|
|
await stop_scheduler()
|
|
|
|
asyncio.run(main())
|