Crawly Config for WAFs and JL File Writer

config.exs

config :revo,
       :user_agent,
       "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"

config :revo, :wait_intervals, [30_000, 45_000, 60_000, 65_000, 76_000]

config :crawly,
  concurrent_requests_per_domain: 1,
  closespider_timeout: 1,
  manager_operations_timeout: 5 * 60_000,
  middlewares: [
    Crawly.Middlewares.DomainFilter,
    Crawly.Middlewares.UniqueRequest,
    {Crawly.Middlewares.UserAgent,
     user_agents: [
       # "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)",
       "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:84.0) Gecko/20100101 Firefox/84.0"
     ]},
    {Crawly.Middlewares.RequestOptions, [timeout: 16_000]}
  ],
  pipelines: [
    Crawly.Pipelines.JSONEncoder,
    {Crawly.Pipelines.WriteToFile, folder: "/app/priv/data", extension: "jl"}
  ]