czechvr scene identifcation by filename

crontab.sh

#!/bin/bash
# install in `crontab -e` as: @daily  $HOME/cvr_id_gist/crontab.sh

cd "${0%/*}"
./update_czechvr_csv.py && git add . && git commit -m "updates $(date +%Y-%m-%d)"

czechvr.csv

czechvr_id.py

"""
czechvr_id - scraper for stashapp

Install:
- put this .py in config/scrapers directory
- also create new file navr_url.yml with following contents:
```
name: "czechvr_id"
sceneByFragment:
  action: script
  script:
    - python3
    - czechvr_id.py
```
- reload scrapers
"""
from urllib.request import Request, urlopen
from base64 import b64decode
import time
import json
import sys
import re
import os


def format_rfc1123(timestamp):
    return time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(timestamp))


def download_if_expired(url, path, maxage=60*30):
    headers = {}
    if os.path.exists(path):
        mtime = os.path.getmtime(path)
        headers["If-Modified-Since"] = format_rfc1123(mtime)

    if not os.path.exists(path) or time.time() - mtime > maxage:
        r = urlopen(Request(url, None, {}))
        if r.status == 200:
            with open(path, 'wb') as fh:
                fh.write(r.read())


FRAGMENT = json.loads(sys.stdin.read())
# print(FRAGMENT, file=sys.stderr)
TITLE = FRAGMENT.get("title")
_id, studio = [s.strip() for s in TITLE.lower().split("-")[0:2]]

studios = ["czechvr", "czechvrcasting", "czechvrfetish"]

if studio not in studios:
    print("== unknown studio %s" % studio, file=sys.stderr)
    print(json.dumps({}))
    sys.exit(0)

CACHE_DIR = os.getenv("STASH_CACHE", ".")
CVR_URL = "https://gist.github.com/poontology/c830392b96dacde0aa39ed910104cb83/raw/%s.csv" % studio
CSVFILE = CACHE_DIR + "/%s.csv" % studio
download_if_expired(CVR_URL, CSVFILE)
lst = [l.split(",") for l in open(CSVFILE).read().split("\n")]

slug = "%s-%s" % (_id, studio)
match = list(filter(lambda s: s[0] == slug, lst))

print("== %s matches %s" % (slug, match), file=sys.stderr)

if not match:
    print(json.dumps({}))
    sys.exit(0)

URL_BASE = "https://www.%s.com/" % studio
res = {
    "url": URL_BASE + "detail-" + match[0][1],
    "title": b64decode(match[0][2]).decode(),
    "image": URL_BASE + "category/%s-%s//%s-big.jpg" % (match[0][1], match[0][3], match[0][0])
}
print(json.dumps(res))

czechvrcasting.csv

czechvrfetish.csv

update_czechvr_csv.py

#!/usr/bin/env python3
"""
update_czechvr_csv - crawls scene index and collects URL slug to id mappings
2022, PoKeH
CzechVR video filenames typically start with a scene number but to get to
the metainfo on the webpage also the id number needs to be known
"""

from urllib.request import Request, urlopen
from urllib.parse import urlparse
from operator import itemgetter
from base64 import b64encode
from random import random
from lxml import html
import time
import re


DEFAULT_UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.127 Safari/537.36"

text = lambda e: e.xpath('normalize-space(text())') if type(e)!=list else list(map(text, e))


def retrieve(url, headers={}):
    if 'User-Agent' not in headers:
        headers['User-Agent'] = DEFAULT_UA
    page = urlopen(Request(url, None, headers))
    return html.parse(page).getroot()


def getscenes(czechvr_page_url):
    print(czechvr_page_url)
    res = retrieve(czechvr_page_url)
    ro = []
    for i,t in enumerate(res.cssselect('article .post')):
        d = {
            'path': t.cssselect('.link a')[0].attrib['href'],
            'title': text(t.cssselect('h2')[0]),
            'picture': t.cssselect('dl8-video')[0].attrib['poster'],
        }
        d['id'] = d['path'].split('-',1)[1]
        d['slug'] = re.sub(r'.*/(.*)-big.jpg', r'\1', d['picture'])
        d['category'] = re.sub(r".*-([^-]+-[^-/]+)/?/(.*)-big.jpg", r'\1', d['picture'])
        d['enctitle'] = b64encode(d["title"].encode()).decode()
        ro.append(d)
    ro = sorted(ro, key=itemgetter('id'))
    return ro


def getlastpage(czechvr_url):
    res = retrieve(czechvr_url)
    el = res.cssselect('a.last')[-1]
    return int(el.attrib['href'].split("=")[-1])


def update_csv(site):
    filename = site + ".csv"
    csv_data = open(filename).read().strip()
    scene_lst = [l.split(",") for l in csv_data.split('\n') if "," in l]
    last_page = getlastpage("http://www.%s.com/" % site)
    items_per_page = 8
    max_pg = int(last_page - len(scene_lst))
    # last_id = int(scene_lst[-1][1])
    scene_dic = {d[1]:d[0] for d in scene_lst if len(d)>1}

    for pg in range(max_pg, 0, -items_per_page):
        new_items = []

        for d in getscenes("https://www.%s.com/?next=%i" % (site, pg)):
            if d["id"] in scene_dic: continue
            new_items.append("%(slug)s,%(id)s,%(enctitle)s,%(category)s" % d)

        if new_items:
            with open(filename, "a") as fh:
                fh.write("\n".join(new_items) + "\n")
                print("pg %i had %i new items" % (pg, len(new_items)))
                time.sleep(random())


if __name__ == "__main__":
    sites = ["czechvr", "czechvrcasting", "czechvrfetish"]
    for site in sites:
        update_csv(site)