fly-python/config.yml at develop · yuandongx/fly-python · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
# ============================================
# fly-python 配置文件 (兜底)
# 环境变量优先级高于本文件
# ============================================

database:
  # 连接 URI (省略时从 host/port/user/password/db 拼接)
  # uri: "mongodb://root:qwerASDF1234@mongo:27017/fly_python"

  host: "120.48.130.105"
  port: 27017
  user: "root"
  password: "qwerASDF1234"
  db: "fly_python"

  # ---- 连接池 (最小化资源) ----
  max_pool_size: 10          # 最大连接数
  min_pool_size: 0           # 最小连接数 (0 = 空闲时不保活)
  max_idle_time_ms: 60000    # 空闲连接最大存活时间 (ms)

  # ---- 超时 ----
  server_timeout_ms: 5000
  connect_timeout_ms: 5000

  # ---- 读写关注 ----
  w: 1
  read_preference: "primaryPreferred"

# ============================================
# 日志配置
# ============================================
logging:
  level: "INFO"                   # DEBUG / INFO / WARNING / ERROR / CRITICAL
  file: "logs/app.log"            # 日志文件路径
  max_bytes: 10485760             # 单个文件最大字节数 (10MB)
  backups: 5                      # 备份文件保留数量
  console: true                   # 是否输出到终端
  file_out: true                  # 是否输出到文件
  format: "%(asctime)s | %(levelname)-8s | %(name)s | %(filename)s:%(lineno)d | %(message)s"
  date_format: "%Y-%m-%d %H:%M:%S"

# ============================================
# Celery 配置 (中间件 Redis, 结果存 MongoDB)
# ============================================
celery:
  # ---- Broker (中间件: Redis) ----
  broker_url: "redis://redis:6379/0"

  # ---- Result Backend (结果: MongoDB) ----
  result_backend: "mongodb://root:qwerASDF1234@mongo:27017/fly_python?authSource=admin"

  # 结果过期时间 (秒), 默认 1 小时
  result_expires: 3600

  # 任务序列化
  task_serializer: "json"
  result_serializer: "json"
  accept_content: ["json"]

  # 时区
  timezone: "Asia/Shanghai"
  enable_utc: false

  # worker 并发 & 预取
  worker_concurrency: 4
  worker_prefetch_multiplier: 1

  # 重试
  task_acks_late: true
  task_reject_on_worker_lost: true

  # ---------- MongoDB Backend 专用 ----------
  mongodb_backend_settings:
    database: "fly_python"
    taskmeta_collection: "celery_taskmeta"
    max_pool_size: 5

  # ---------- 定时任务 (Beat Schedule) ----------
  beat_schedule:
    # # 10s钟执行一次
    # task_get_sina_data:
    #   task: "app.spider.tasks.task_get_sina_data"
    #   schedule: 10.0

    # 9点30至11点30每分钟执行一次
    task_get_sina_data_morning:
      task: "app.spider.tasks.task_get_sina_data"
      schedule:
        hour: "9-11"
        minute: 0
        day_of_week: "1-5"  # 仅工作日执行
    # 13点至15点每分钟执行一次
    task_get_sina_data_afternoon:
      task: "app.spider.tasks.task_get_sina_data"
      schedule:
        hour: "13-15"
        minute: 0
        day_of_week: "1-5"  # 仅工作日执行
    # 9点30至11点30每分钟执行一次
    task_get_fund_rank_data_morning:
      task: "app.spider.tasks.task_get_fund_rank_data"
      schedule:
        hour: "9-11"
        minute: "*/30" # 每30分钟执行一次
        day_of_week: "1-5"  # 仅工作日执行
    # 13点至15点每分钟执行一次
    task_get_fund_rank_data_afternoon:
      task: "app.spider.tasks.task_get_fund_rank_data"
      schedule:
        hour: "13-16"
        minute: "*/30" # 每30分钟执行一次
        day_of_week: "1-5"  # 仅工作日执行


    # 09点至15点每分钟执行一次
    task_get_sina_zs_data:
      task: "app.spider.tasks.task_get_sina_zs_data"
      schedule:
        hour: "9-15"
        minute: "*/2" # 每2分钟执行一次
        day_of_week: "1-5"  # 仅工作日执行