Order Data (Minutes/Noon/Namshi)

orders.md

#  TODO:
#  - Adjust Timezones
#  - Revise Batch Numbers & Batched Orders
#  - Order Value/Volume/Weight
#  - COD vs Prepaid
#  - MP sales_order tables (Minutes, Noon, Namshi)
#    + Estimates
#  - Reasons:
#    + Undelivered
#    + Cancelled
#    + Delivery Job
#  - OOS Numbers
#  - Dropoff Leg:
#    + Driver username/usercode
#    + unassigned_count
#    + id_delivery_job
#    + id_delivery_job_line
#    + is_batched
#    + batch_count
#    + batch_count_unassigned
#    + batch_seq_[scanned|arrived|delivered]
#    + batch_orders_others
#  - Batching Recommendations

SELECT
  ANY_VALUE(w.code) wh_code,
  ANY_VALUE(w.partner_warehouse_code) wh_code_partner,
  ANY_VALUE(CONCAT(w.area, ' (', w.city, ')')) wh_name,
  ANY_VALUE(m.code) marketplace,
  ANY_VALUE(dm.code) mode,
  ANY_VALUE(IF(dt.code = 'dropoff_customer_delivery', 'dropoff', 'direct')) type, -- dropoff_customer_delivery vs direct_customer_delivery
  ANY_VALUE(o.id_mp_order) id_order,
  o.order_nr,
  COUNT(DISTINCT i.catalog_sku) sku_count,
  COUNT(DISTINCT i.item_nr) item_count,
  ANY_VALUE(pu.username) picker_username, -- Username for picker as defined on HOMS
  ANY_VALUE(pu.email) picker_usercode, -- IDP user_code for picker (if order is not cancelled before picking starts)
  COUNT(DISTINCT s.awb_nr) package_count, -- Order package count
  STRING_AGG(DISTINCT s.awb_nr) package_awbs, -- Order package awbs
  ANY_VALUE(du.username) driver_username, -- Username for the driver (for Rocket/Minutes orders)
  ANY_VALUE(du.email) driver_usercode, -- IDP user_code for driver (if order is scanned/picked_up by a driver)
  COUNT(DISTINCT IF(dl.is_manually_unassigned = 1, dl.id_delivery_job_line, NULL)) unassigned_count, -- Number of times order was unassigned

  ANY_VALUE(IF(dl.is_manually_unassigned = 0, d.id_delivery_job, NULL)) id_delivery_job, -- Delivery Job ID for final delivery job (useful for getting average batch size)
  ANY_VALUE(IF(dl.is_manually_unassigned = 0, dl.id_delivery_job_line, NULL)) id_delivery_job_line, -- Delivery Job Line ID for final delivery job


  COUNT(DISTINCT IF(dlwh.is_manually_unassigned = 0, dlwh.id_mp_order, NULL)) > 1 is_batched, -- Was the order batched with other orders (in final delivery job)
  COUNT(DISTINCT IF(dlwh.is_manually_unassigned = 0, dlwh.id_mp_order, NULL)) batch_count, -- Number of orders in the batch (in final delivery job)
  COUNT(DISTINCT IF(dlwh.is_manually_unassigned = 1, dlwh.id_mp_order, NULL)) batch_count_unassigned, -- Number of orders scanned in the batch that were unassigned between 'scanned' and 'delivered'
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.seq_scanned, NULL)) batch_seq_scanned, -- Sequence this order was scanned within batch
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.seq_arrived, NULL)) batch_seq_arrived, -- Sequence this order has arrived within batch
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.seq_delivered, NULL)) batch_seq_delivered, -- Sequence this order was delivered within batch
  STRING_AGG(DISTINCT IF(dlwh.id_mp_order IS NULL OR dlwh.is_manually_unassigned = 1 OR dlwh.id_mp_order = o.id_mp_order, NULL, o.order_nr)) batch_orders_other, -- Other batched orders (if any, excluding unassigned orders)

  ANY_VALUE(o.is_cancelled) is_cancelled, -- Was order cancelled?
  MIN(ch.modified_at) cancelled_at, -- When was the order cancelled (in HOMS)

  ANY_VALUE(o.estimated_pickup_at) estimated_pickup_at,
  ANY_VALUE(o.estimated_delivery_at) estimated_delivery_at,
  ANY_VALUE(COALESCE(o_instant.created_at, o_noon.created_at, o_namshi.created_at, NULL)) placed_at, -- When was order pushed on Marketplace
  ANY_VALUE(oh.created) pushed_at, -- When was order pushed to HOMS?
  ANY_VALUE(ph.pending) picking_job_created_at, -- When was picking job created?
  ANY_VALUE(ph.picking) picking_at, -- When did picking start?
  ANY_VALUE(ph.packing) packing_at, -- When did packing start?
  ANY_VALUE(ph.completed) fulfilled_at, -- When did picking/packing end?
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.scanned, NULL)) scanned_at, -- When was order scanned (in final delivery job)?
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.picked_up, NULL)) picked_up_at, -- When was order picked_up (in final delivery job)?
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.arrived_at_delivery, NULL)) arrived_at, -- When was order arrived_at_delivery (in final delivery job)?
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.delivered, NULL)) delivered_at, -- When was order delivered (in final delivery job)?
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.undelivered, NULL)) undelivered_at, -- When was order undelivered (in final delivery job)?
  ANY_VALUE(IF(dlwh.id_mp_order = o.id_mp_order, dlwh.returned, NULL)) undelivered_returned_at, -- When was an undelivered order marked as returned (in final delivery job)?
  ANY_VALUE(dh.completed) delivery_completed_at, -- When was the driver back to warehouse after delivery/undelivery of orders in the batch?

  ANY_VALUE(CONCAT(w.latitude, ',', w.longitude)) warehouse_location,
  ANY_VALUE(CONCAT(w.waiting_loc_latitude, ',', w.waiting_loc_latitude)) parking_location,
  ANY_VALUE(lh.picked_up) picked_up_location,
  ANY_VALUE(lh.arrived_at_delivery) arrived_location,
  ANY_VALUE(lh.delivered) delivered_location,
  ANY_VALUE(lh.undelivered) undelivered_location,
  ANY_VALUE(lh.returned) returned_location,

FROM `noondwh.schoms_homs.mp_order` o
JOIN `noondwh.schoms_homs.warehouse` w ON (w.partner_warehouse_code = o.warehouse_code)
JOIN `noondwh.schoms_homs.country` c ON (c.id_country = w.id_country)
JOIN `noondwh.schoms_homs.mp` m USING (id_mp)
JOIN `noondwh.schoms_homs.delivery_type` dt USING (id_delivery_type)
JOIN `noondwh.schoms_homs.delivery_mode` dm USING (id_delivery_mode)
JOIN `noondwh.schoms_homs.mp_order_item` i USING (id_mp_order)
JOIN `noondwh.schoms_homs.picking_job` p ON (p.id_mp_order = o.id_mp_order) -- Picking Job (All orders have picking jobs, hence the JOIN)

LEFT JOIN `noondwh.instant_instant_order.sales_order` o_instant ON (o_instant.order_nr = o.order_nr AND m.code = 'nooninstant')
LEFT JOIN `noondwh.sales.sales_order` o_noon ON (o_noon.order_nr = REGEXP_REPLACE(o.order_nr, '-(R|E|ORDER1)$', '') AND m.code = 'noon')
LEFT JOIN `noondwh.nmnamshi_namshi_order.sales_order` o_namshi ON (o_namshi.order_nr = REGEXP_REPLACE(o.order_nr, '-(R|E)$', '') AND m.code = 'namshi_v2')

LEFT JOIN `noondwh.schoms_homs.user` pu ON (pu.id_user = p.id_user) -- Picker (Note: not all picking jobs get picked, some get cancelled before picking, hence the LEFT JOIN)
LEFT JOIN `noondwh.schoms_homs.shipment` s ON (s.id_mp_order = o.id_mp_order) -- Shipments

LEFT JOIN `noondwh.schoms_homs.delivery_job_line` dl ON (dl.id_mp_order = o.id_mp_order) -- Delivery job line(s) for order (including ones that were unassigned)
LEFT JOIN `noondwh.schoms_homs.delivery_job` d ON (d.id_delivery_job = dl.id_delivery_job AND dl.is_manually_unassigned = 0) -- Main/final delivery job for order
LEFT JOIN `noondwh.schoms_homs.user` du ON (du.id_user = d.id_user) -- Driver (for main/final delivery job)

-- Cancellation History: We get the order cancellation a bit differently than other order statuses, as order cancellation is not implemented as a status, but rather as an is_cancelled flag.
LEFT JOIN `noondwh.schoms_homs.mp_order_history` ch ON (ch.id_mp_order = o.id_mp_order AND ch.is_cancelled = 1)

-- Order History (Note: we can also INNER JOIN)
LEFT JOIN (
  SELECT * FROM (
    -- Statuses: created, fulfilling, fulfilled, in_transit, dropped_off, picked_up, arrived_at_delivery, delivered, undelivered, returned
    -- Outdated: cancelled (2022-08-04), reserved (2022-02-28)
    SELECT id_mp_order, code status, modified_at -- SELECT DISTINCT code
    FROM `noondwh.schoms_homs.mp_order_history`
    JOIN `noondwh.schoms_homs.status` USING (id_status)
  ) PIVOT (MIN(modified_at) FOR status IN ( 'created', 'fulfilling', 'fulfilled', 'in_transit', 'dropped_off', 'picked_up', 'arrived_at_delivery', 'delivered', 'undelivered', 'returned' ))
) oh ON (oh.id_mp_order = o.id_mp_order)

-- Picking Job History (Note: we can also INNER JOIN)
LEFT JOIN (
  SELECT * FROM (
    -- Statuses: pending, picking, packing, completed, cancelled
    SELECT id_picking_job, code status, modified_at -- SELECT DISTINCT code
    FROM `noondwh.schoms_homs.picking_job_history`
    JOIN `noondwh.schoms_homs.status` USING (id_status)
  ) PIVOT (MIN(modified_at) FOR status IN ( 'pending', 'picking', 'packing', 'completed', 'cancelled' ))
) ph ON (ph.id_picking_job = p.id_picking_job)

LEFT JOIN ( -- Order Delivery Location History
  SELECT * FROM (
    -- Statuses: in_transit, dropped_off, picked_up, arrived_at_delivery, delivered, undelivered, returned
    SELECT id_mp_order, code status, CONCAT(latitude, ',', longitude) location -- SELECT DISTINCT code
    FROM `noondwh.schoms_homs.order_delivery_location_history`
    JOIN `noondwh.schoms_homs.status` USING (id_status)
  ) PIVOT (ANY_VALUE(location) FOR status IN ( 'in_transit', 'dropped_off', 'picked_up', 'arrived_at_delivery', 'delivered', 'undelivered', 'returned' ))
) lh ON (lh.id_mp_order = o.id_mp_order)

LEFT JOIN ( -- Delivery Job History
  SELECT * FROM (
    -- Statuses: assigning, delivering, returning, completed
    SELECT id_delivery_job, code status, modified_at -- SELECT DISTINCT code
    FROM `noondwh.schoms_homs.delivery_job_history`
    JOIN `noondwh.schoms_homs.status` USING (id_status)
  ) PIVOT (MIN(modified_at) FOR status IN ( 'assigning', 'delivering', 'returning', 'completed' ))
) dh ON (dh.id_delivery_job = d.id_delivery_job)

-- Delivery Job Line With History: All delivery job lines for current delivery job (for stats on current batch)
LEFT JOIN (
  SELECT
    *,
    ROW_NUMBER() OVER (PARTITION BY IF(is_manually_unassigned = 0, id_delivery_job, NULL) ORDER BY scanned ASC) seq_scanned,             -- Sequence of orders scanned, excluding unassigned
    ROW_NUMBER() OVER (PARTITION BY IF(is_manually_unassigned = 0, id_delivery_job, NULL) ORDER BY arrived_at_delivery ASC) seq_arrived, -- Sequence of orders arrived, excluding unassigned
    ROW_NUMBER() OVER (PARTITION BY IF(is_manually_unassigned = 0, id_delivery_job, NULL) ORDER BY delivered ASC) seq_delivered,         -- Sequence of orders delivered, excluding unassigned
  FROM `noondwh.schoms_homs.delivery_job_line` dl
  LEFT JOIN (
    SELECT * FROM (
      -- Statuses: scanned, picked_up, arrived_at_delivery, delivered, undelivered, returned
      SELECT id_delivery_job_line, code status, modified_at -- SELECT DISTINCT code
      FROM `noondwh.schoms_homs.delivery_job_line_history`
      JOIN `noondwh.schoms_homs.status` USING (id_status)
    ) PIVOT (MIN(modified_at) FOR status IN ( 'scanned', 'picked_up', 'arrived_at_delivery', 'delivered', 'undelivered', 'returned' ))
  ) dlh ON (dlh.id_delivery_job_line = dl.id_delivery_job_line AND dl.is_manually_unassigned = 0) -- We're not interested in history for unassigned lines
) dlwh ON (dlwh.id_delivery_job = d.id_delivery_job)

WHERE TRUE
-- AND c.code IN ('ae') -- ae, sa, eg
-- AND w.is_core_warehouse = 1
-- AND w.is_nim_warehouse = 1
-- AND m.code IN ('noon', 'namshi_v2') -- nooninstant, noon, namshi_v2
-- AND dm.code IN ('rocket', 'express') -- minutes, rocket, express
-- AND dt.code IN ('direct', 'dropoff') -- direct, dropoff
-- AND DATE(o.created_at) > DATE_SUB(CURRENT_DATE(), INTERVAL 2 DAY)
-- AND DATE(o.created_at) < CURRENT_DATE()
GROUP BY o.order_nr
ORDER BY id_order DESC;