', '<'),
"uploader" : extr('
', '
'),
"date" : self.parse_datetime_iso(extr(
'>Posted:
', ' | ')),
"parent" : extr(
'>Parent:
Visible: | ', '<'),
"language" : extr('>Language: | ', ' '),
"filesize" : text.parse_bytes(extr(
'>File Size: | ', '<').rstrip("Bbi")),
"filecount" : extr('>Length: | ', ' '),
"favorites" : extr('id="favcount">', ' '),
"rating" : extr(">Average: ", "<"),
"torrentcount" : extr('>Torrent Download (', ')'),
}
uploader = data["uploader"]
if uploader and uploader[0] == "<":
data["uploader"] = text.unescape(text.extr(uploader, ">", "<"))
f = data["favorites"][0]
if f == "N":
data["favorites"] = "0"
elif f == "O":
data["favorites"] = "1"
data["lang"] = util.language_to_code(data["language"])
data["tags"] = [
text.unquote(tag.replace("+", " "))
for tag in text.extract_iter(page, 'hentai.org/tag/', '"')
]
return data
def metadata_from_api(self):
data = {
"method" : "gdata",
"gidlist" : ((self.gallery_id, self.gallery_token),),
"namespace": 1,
}
data = self.request_json(self.api_url, method="POST", json=data)
if "error" in data:
raise exception.AbortExtraction(data["error"])
return data["gmetadata"][0]
def image_from_page(self, page):
"""Get image url and data from webpage"""
pos = page.index('= 0:
origurl, pos = text.rextract(i6, '"', '"', pos)
url = text.unescape(origurl)
data = self._parse_original_info(text.extract(
i6, "ownload original", "<", pos)[0])
data["_fallback"] = self._fallback_original(nl, url)
else:
url = imgurl
data = self._parse_image_info(url)
data["_fallback"] = self._fallback_1280(
nl, request["page"], imgkey)
except IndexError:
self.log.debug("Page content:\n%s", page)
raise exception.AbortExtraction(
f"Unable to parse image info for '{url}'")
data["num"] = request["page"]
data["image_token"] = imgkey
data["_url_1280"] = imgurl
data["_nl"] = nl
self._check_509(imgurl)
yield url, text.nameext_from_url(url, data)
request["imgkey"] = nextkey
def images_from_mpv(self):
"""Get image url and data from MPV"""
url = f"{self.root}/mpv/{self.gallery_id}/{self.gallery_token}/"
page = self.request(url).text
images = util.json_loads(text.extr(page, "var imagelist = ", ";"))
api_url = self.api_url
pnum = self.image_num - 1
request = {
"method": "imagedispatch",
"gid" : self.gallery_id,
"page" : 0,
"imgkey": "",
"mpvkey": text.extr(page, 'var mpvkey = "', '"'),
}
if pnum:
images = util.advance(images, pnum)
for image in images:
pnum += 1
request["page"] = pnum
request["imgkey"] = imgkey = image["k"]
info = self.request_json(api_url, method="POST", json=request)
try:
imgurl = info["i"]
if self.original and info.get("o") and " " in info["o"]:
url = f"{self.root}/{info['lf']}"
try:
data = self._parse_mpv_info(info)
except ValueError:
self.log.warning(
"Failed to extract original file metadata")
data = self._parse_image_info(imgurl)
data["_fallback"] = self._fallback_mpv_original(info)
else:
url = imgurl
data = self._parse_image_info(url)
data["_fallback"] = self._fallback_mpv_1280(info, request)
except IndexError:
self.log.debug("Page content:\n%s", info)
raise exception.AbortExtraction(
f"Unable to parse image info for '{url}'")
data["num"] = pnum
data["_nl"] = info["s"]
data["_url_1280"] = imgurl
data["image_token"] = imgkey
self._check_509(imgurl)
if name := image.get("name"):
text.nameext_from_name(name, data)
else:
text.nameext_from_url(url, data)
yield url, data
def _validate_response(self, response):
if response.history or not response.headers.get(
"content-type", "").startswith("text/html"):
return True
page = response.text
self.log.warning("'%s'", page)
if " requires GP" in page:
gp = self.config("gp")
if gp == "stop":
raise exception.AbortExtraction("Not enough GP")
elif gp == "wait":
self.input("Press ENTER to continue.")
return response.url
self.log.info("Falling back to non-original downloads")
self.original = False
return self.data["_url_1280"]
if " temporarily banned " in page:
raise exception.AuthorizationError("Temporarily Banned")
self._limits_exceeded()
return response.url
def _validate_signature(self, signature):
"""Return False if all file signature bytes are zero"""
if signature:
if byte := signature[0]:
# 60 == b"<"
if byte == 60 and b"", "").replace(",", "")
self.log.debug("Image Limits: %s/%s", current, self.limits)
self._limits_remaining = self.limits - text.parse_int(current)
return page
def _check_509(self, url):
# full 509.gif URLs
# - https://exhentai.org/img/509.gif
# - https://ehgt.org/g/509.gif
if url.endswith(("hentai.org/img/509.gif",
"ehgt.org/g/509.gif")):
self.log.debug(url)
self._limits_exceeded()
def _limits_exceeded(self):
msg = "Image limit exceeded!"
action = self.config("limits-action")
if not action or action == "stop":
ExhentaiExtractor.LIMIT = True
raise exception.AbortExtraction(msg)
self.log.warning(msg)
if action == "wait":
self.input("Press ENTER to continue.")
self._limits_update()
elif action == "reset":
self._limits_reset()
else:
self.log.error("Invalid 'limits-action' value '%s'", action)
def _limits_check(self, data):
if not self._limits_remaining or data["num"] % 25 == 0:
self._limits_update()
self._limits_remaining -= data["cost"]
if self._limits_remaining <= 0:
self._limits_exceeded()
def _limits_reset(self):
self.log.info("Resetting image limits")
self._request_home(
method="POST",
headers={"Content-Type": "application/x-www-form-urlencoded"},
data=b"reset_imagelimit=Reset+Quota")
_limits_update = _request_home
def _gallery_page(self):
url = f"{self.root}/g/{self.gallery_id}/{self.gallery_token}/"
response = self.request(url, fatal=False)
page = response.text
if response.status_code == 404 and "Gallery Not Available" in page:
raise exception.AuthorizationError()
if page.startswith(("Key missing", "Gallery not found")):
raise exception.NotFoundError("gallery")
if page.count("hentai.org/mpv/") > 1:
if self.gallery_token is None:
raise exception.AbortExtraction(
"'/s/' URLs in MPV mode are not supported")
self.mpv = True
return page
def _image_page(self):
url = (f"{self.root}/s/{self.image_token}"
f"/{self.gallery_id}-{self.image_num}")
page = self.request(url, fatal=False).text
if page.startswith(("Invalid page", "Keep trying")):
raise exception.NotFoundError("image page")
return page
def _fallback_original(self, nl, fullimg):
url = f"{fullimg}?nl={nl}"
for _ in util.repeat(self.fallback_retries):
yield url
def _fallback_mpv_original(self, info):
url = f"{self.root}/{info['lf']}?nl={info['s']}"
for _ in util.repeat(self.fallback_retries):
yield url
def _fallback_1280(self, nl, num, token=None):
if not token:
token = self.key_start
for _ in util.repeat(self.fallback_retries):
url = f"{self.root}/s/{token}/{self.gallery_id}-{num}?nl={nl}"
page = self.request(url, fatal=False).text
if page.startswith(("Invalid page", "Keep trying")):
return
url, data = self.image_from_page(page)
yield url
nl = data["_nl"]
def _fallback_mpv_1280(self, info, request):
for _ in util.repeat(self.fallback_retries):
request["nl"] = info["s"]
info = self.request_json(self.api_url, method="POST", json=request)
yield info["i"]
def _parse_image_info(self, url):
for part in url.split("/")[4:]:
try:
_, size, width, height, _ = part.split("-")
break
except ValueError:
pass
else:
size = width = height = 0
return {
"cost" : 1,
"size" : text.parse_int(size),
"width" : text.parse_int(width),
"height": text.parse_int(height),
}
def _parse_original_info(self, info):
parts = info.lstrip().split(" ")
size = text.parse_bytes(parts[3] + parts[4][0])
return {
# 1 initial point + 1 per 0.1 MB
"cost" : 1 + math.ceil(size / 100_000),
"size" : size,
"width" : text.parse_int(parts[0]),
"height": text.parse_int(parts[2]),
}
def _parse_mpv_info(self, info):
_, _, w, _, h, s, u, _ = info["o"].split()
size = text.parse_bytes(s + u[0])
return {
# 1 initial point + 1 per 0.1 MB
"cost" : 1 + math.ceil(size / 100_000),
"size" : size,
"width" : text.parse_int(w),
"height": text.parse_int(h),
}
class ExhentaiSearchExtractor(ExhentaiExtractor):
"""Extractor for exhentai search results"""
subcategory = "search"
pattern = BASE_PATTERN + r"/(?:\?([^#]*)|tag/([^/?#]+))"
example = "https://e-hentai.org/?f_search=QUERY"
def __init__(self, match):
ExhentaiExtractor.__init__(self, match)
_, query, tag = self.groups
if tag:
if "+" in tag:
ns, _, tag = tag.rpartition(":")
tag = f"{ns}:\"{tag.replace('+', ' ')}$\""
else:
tag += "$"
self.params = {"f_search": tag, "page": 0}
else:
self.params = text.parse_query(query)
if "next" not in self.params:
self.params["page"] = text.parse_int(self.params.get("page"))
def _init(self):
self.search_url = self.root
def items(self):
self.login()
data = {"_extractor": ExhentaiGalleryExtractor}
search_url = self.search_url
params = self.params
while True:
last = None
page = self.request(search_url, params=params).text
for match in ExhentaiGalleryExtractor.pattern.finditer(page):
url = match[0]
if url == last:
continue
last = url
data["gallery_id"] = text.parse_int(match[2])
data["gallery_token"] = match[3]
yield Message.Queue, url + "/", data
next_url = text.extr(page, 'nexturl="', '"', None)
if next_url is not None:
if not next_url:
return
search_url = next_url
params = None
elif 'class="ptdd">><' in page or ">No hits found" in page:
return
else:
params["page"] += 1
class ExhentaiFavoriteExtractor(ExhentaiSearchExtractor):
"""Extractor for favorited exhentai galleries"""
subcategory = "favorite"
pattern = BASE_PATTERN + r"/favorites\.php(?:\?([^#]*)())?"
example = "https://e-hentai.org/favorites.php"
def _init(self):
self.search_url = self.root + "/favorites.php"
|