pulpie-eww.el

replacing eww-readable with pulpie-orange-small
Log | Files | Refs | LICENSE

commit 2d76af7f779fdf79a9a87bbc58b86789d79062a7
Author: Vineet Kumar <git@vineetk.net>
Date:   Fri, 20 Nov 2026 01:11:19 +0100

initial payload

the script was made by qwen3.8-flash-next, with quite some steering by
me. I know it's ugly.

Diffstat:
Apulpie.el | 363+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 363 insertions(+), 0 deletions(-)

diff --git a/pulpie.el b/pulpie.el @@ -0,0 +1,363 @@ +;;; pulpie.el --- eww-readable replacement engine, using pulpie BERT as classifier -*- lexical-binding: t; -*- + +;; Version: 1.0 +;; Package-Requires: ((emacs "28.1")) + +;;; Commentary: + +;; One entry point: +;; +;; (pulpie-main-html "https://example.com/article") +;; => "<main><h1>...</h1><p>...</p></main>" +;; +;; It returns the page's main content as an HTML fragment - navigation, +;; sidebars, comments and footers dropped - intended as the reader step in +;; front of eww, which otherwise shows the whole document. +;; +;; Classification runs on a llama-server serving a pulpie GGUF: +;; +;; llama-server -m pulpie-orange-small-cls-f16.gguf --embeddings \ +;; -b 8192 -ub 8192 -c 8192 --port 8888 +;; +;; How it works: the page is split into leaf blocks, packed into the model's +;; chunk template +;; +;; [BOS] block0 <|sep|> block1 <|sep|> ... <|sep|> [EOS] +;; +;; and every <|sep|> row of /embedding carries that block's [other, main] +;; logits. Token ids come from the server's /tokenize endpoint, so they match +;; the tokenizer baked into the GGUF exactly. +;; +;; Blocks are classified as leaves but emitted as whole subtrees: each kept +;; block is lifted to the largest ancestor containing only kept blocks, so +;; <ul>, <table> and section wrappers survive. + +;;; Code: + +(require 'cl-lib) +(require 'dom) +(require 'json) +(require 'pcase) +(require 'url) +(require 'url-http) + +(defgroup pulpie nil + "Main-content extraction with a pulpie GGUF behind llama-server." + :group 'hypermedia) + +;; (defcustom pulpie-server "http://localhost:8888" +;; "Base URL of the llama-server serving a pulpie GGUF." +;; :type 'string :group 'pulpie) +(defcustom pulpie-server "http://llm.demiurge.epistemia/upstream/pulpie-orange-small" + "Base URL of the llama-server serving a pulpie GGUF." + :type 'string :group 'pulpie) + +(defcustom pulpie-max-tokens 8192 + "Token budget per request; must not exceed the server's -b/-ub/-c value." + :type 'integer :group 'pulpie) + +(defconst pulpie-bos "<|begin_of_text|>" "Chunk prefix.") +(defconst pulpie-eos "<|end_of_text|>" "Chunk suffix.") +(defconst pulpie-sep "<|sep|>" "Block separator.") +(defconst pulpie-sep-token-id 128256 "Vocab id of <|sep|>.") + +(defconst pulpie-block-tags + '(p li h1 h2 h3 h4 h5 h6 pre blockquote figcaption td th dd dt summary) + "Block-level tags whose text is classified (leaf elements only).") + +(defconst pulpie-noise-tags + '(script style noscript iframe template svg canvas form) + "Tags dropped when merged blocks are serialized.") + +(defconst pulpie-void-tags + '("area" "base" "br" "col" "embed" "hr" "img" "input" "link" "meta" + "source" "track" "wbr") + "HTML elements that must not be printed with a closing tag.") + +;;; HTTP + +(defun pulpie--header-end () + "Return the position just past the HTTP header block." + (save-excursion + (goto-char (point-min)) + (if (re-search-forward "\r?\n\r?\n" nil t) (match-end 0) (point-min)))) + +(defun pulpie--body (buffer url) + "Return BUFFER's decoded body as a string, then kill BUFFER. +URL is only used in the error message. Signals on a 4xx/5xx status." + (with-current-buffer buffer + (unwind-protect + (let* ((status (save-excursion + (goto-char (point-min)) + (when (looking-at "HTTP/[0-9.]+ \\([0-9]+\\)") + (string-to-number (match-string 1))))) + (header-end (pulpie--header-end)) + (charset (save-excursion + (goto-char (point-min)) + (when (re-search-forward + "^Content-Type:.*?charset=\\([^; \r\n]+\\)" + header-end t) + (match-string 1)))) + ;; no charset is UTF-8, not the ISO-8859-1 of an old RFC + (coding (let ((sym (and charset (ignore-errors (intern charset))))) + (if (and sym (coding-system-p sym)) sym 'utf-8)))) + (when (and status (>= status 400)) + (error "Request to %s failed (%s): %s" + url status + (string-trim + (buffer-substring-no-properties header-end (point-max))))) + (let ((body (buffer-substring-no-properties (pulpie--header-end) (point-max)))) + ;; url.el leaves the body as raw bytes (a unibyte string); decode + ;; them here. decode-coding-region on the buffer is a no-op. + (if (multibyte-string-p body) + body + (decode-coding-string body coding)))) + (kill-buffer buffer)))) + +(defun pulpie--get (url) + "Fetch URL and return its decoded body." + (let ((url-user-agent url-user-agent) + (url-mime-accept-string "text/html,application/xhtml+xml,*/*") + (url-show-status nil) + (url-request-method "GET") + (url-request-extra-headers + (list (cons "User-Agent" pulpie-user-agent) + (cons "Accept" "text/html,application/xhtml+xml,*/*")))) + (pulpie--body (url-retrieve-synchronously url t nil 60) url))) + +(defun pulpie--post-json (url payload) + "POST PAYLOAD as JSON to URL and return the parsed response." + (let* ((url-show-status nil) + (url-request-method "POST") + (url-request-extra-headers + (list (cons "Content-Type" "application/json") + (cons "User-Agent" pulpie-user-agent))) + (url-request-data (encode-coding-string (json-encode payload) 'utf-8))) + (json-parse-string (pulpie--body (url-retrieve-synchronously url t nil 300) url) + ;; keys come back interned, hence `assq' below + :object-type 'alist :array-type 'list))) + +(defun pulpie--tokenize (text) + "Return the token id list of TEXT, from the server's tokenizer." + (cdr (assq 'tokens (pulpie--post-json (concat pulpie-server "/tokenize") + (list (cons "content" text)))))) + +(defun pulpie--logits (text) + "Return TEXT's per-token [other, main] logits as a list of rows." + (let ((first (car (pulpie--post-json (concat pulpie-server "/embedding") + (list (cons "content" text)))))) + (unless first (error "Empty /embedding response")) + (cdr (assq 'embedding first)))) + +;;; DOM -> blocks + +(defun pulpie--element-p (node) + "Non-nil if NODE is a DOM element rather than a text node." + (and (consp node) (symbolp (car node)))) + +(defun pulpie--walk (node fn) + "Call FN with every element under NODE in document order, plus its parent." + (letrec ((go (lambda (n parent) + (funcall fn n parent) + (dolist (child (dom-children n)) + (when (pulpie--element-p child) + (funcall go child n)))))) + (when (pulpie--element-p node) (funcall go node nil)))) + +(defun pulpie--has-block-descendant-p (node) + "Non-nil if any descendant of NODE is one of `pulpie-block-tags'." + (catch 'found + (dolist (child (dom-children node)) + (when (pulpie--element-p child) + (when (memq (dom-tag child) pulpie-block-tags) (throw 'found t)) + (when (pulpie--has-block-descendant-p child) (throw 'found t)))) + nil)) + +(defun pulpie--leaf-block-p (node) + "Non-nil if NODE is a block containing no other block." + (and (memq (dom-tag node) pulpie-block-tags) + (not (pulpie--has-block-descendant-p node)))) + +(defun pulpie--escape (s) + "Escape S for the classifier markup. +Mirrors Python's html.escape(s, quote=True), which is what the model was +verified against - including the apostrophe, because an apostrophe becomes +&#x27; and that changes the token ids the classifier sees." + (let ((s (replace-regexp-in-string "&" "&amp;" s))) + (setq s (replace-regexp-in-string "<" "&lt;" s)) + (setq s (replace-regexp-in-string ">" "&gt;" s)) + (setq s (replace-regexp-in-string "\"" "&quot;" s)) + (replace-regexp-in-string "'" "&#x27;" s))) + +(defun pulpie--collapse (s) + "Collapse every whitespace run in S to a single space." + (string-trim (replace-regexp-in-string "[ \t\r\n]+" " " s))) + +(defun pulpie--page-blocks (html) + "Split HTML into leaf blocks, returning (MARKUP NODES DOM). +MARKUP is what the classifier sees: a div carrying `_item_id' and the block's +raw text, mirroring pulpie's simplify output, since the model was distilled on +that segmentation and not on the page's original tag names. NODES are the +matching DOM elements, used to rebuild HTML around the kept blocks." + (let* ((dom (with-temp-buffer + (insert html) + (libxml-parse-html-region (point-min) (point-max)))) + (root (or (car (dom-by-tag dom 'body)) dom)) + (markup '()) (nodes '())) + (pulpie--walk root + (lambda (n _parent) + (when (pulpie--leaf-block-p n) + (let ((text (string-trim (or (dom-inner-text n) "")))) + (when (>= (length text) 2) + (push (format "<div _item_id=\"%d\">%s</div>" + (length markup) (pulpie--escape text)) + markup) + (push n nodes)))))) + (list (nreverse markup) (nreverse nodes) dom))) + +;;; Labels + +(defun pulpie--argmax (row) + "Return the index of the largest value in ROW." + (let ((best 0) (best-value (car row)) (i 0)) + (dolist (v row) + (when (> v best-value) (setq best i best-value v)) + (setq i (1+ i))) + best)) + +(defun pulpie--label-chunk (chunk) + "Classify CHUNK (a list of block markup strings), returning 0/1 per block." + (let* ((text (concat pulpie-bos + (mapconcat (lambda (b) (concat b pulpie-sep)) chunk "") + pulpie-eos)) + (ids (pulpie--tokenize text)) + (rows (pulpie--logits text)) + (sep-rows '()) (i 0)) + (dolist (tok ids) + (when (= tok pulpie-sep-token-id) (push i sep-rows)) + (setq i (1+ i))) + (setq sep-rows (nreverse sep-rows)) + (unless (= (length sep-rows) (length chunk)) + (error "%d <|sep|> rows for %d blocks" (length sep-rows) (length chunk))) + (mapcar (lambda (row-index) (pulpie--argmax (nth row-index rows))) sep-rows))) + +(defun pulpie--block-sizes (blocks) + "Return each block's token count, from a single /tokenize round trip." + (let ((ids (pulpie--tokenize (mapconcat #'identity blocks pulpie-sep))) + (sizes '()) (start 0) (i 0)) + (dolist (tok ids) + (when (= tok pulpie-sep-token-id) + (push (- i start) sizes) + (setq start (1+ i))) + (setq i (1+ i))) + (nreverse (cons (- (length ids) start) sizes)))) + +(defun pulpie--classify (blocks &optional max-tokens) + "Label every block 1 (main) or 0 (other), one request per chunk." + (let* ((max-tokens (or max-tokens pulpie-max-tokens)) + (sizes (pulpie--block-sizes blocks)) + (labels '()) (chunk '()) (chunk-tokens 0)) + (cl-loop for block in blocks for size in sizes do + ;; tokens so far + this block + one <|sep|> per block + BOS/EOS + (when (and chunk (> (+ chunk-tokens size (length chunk) 3) max-tokens)) + (setq labels (nconc labels (pulpie--label-chunk chunk))) + (setq chunk '() chunk-tokens 0)) + (setq chunk (append chunk (list block)) + chunk-tokens (+ chunk-tokens size))) + (when chunk (setq labels (nconc labels (pulpie--label-chunk chunk)))) + labels)) + +;;; Rebuild + +(defun pulpie--prune (node) + "Return a copy of NODE without `pulpie-noise-tags' descendants." + (if (stringp node) + node + (cons (dom-tag node) + (cons (dom-attributes node) + (let (acc) + (dolist (child (dom-children node)) + (cond ((stringp child) (push child acc)) + ((not (memq (dom-tag child) pulpie-noise-tags)) + (push (pulpie--prune child) acc)))) + (nreverse acc)))))) + +(defun pulpie--self-closing (s) + "Fix up libxml's XHTML-style self-closing tags in S. +libxml prints every empty element as `<foo />', which HTML reads as an +ordinary start tag - unbalanced inside a fragment. Void elements lose the +slash; everything else gets an explicit close." + (replace-regexp-in-string + "<\\([a-zA-Z][a-zA-Z0-9-]*\\)\\( [^>]*\\)?/>" + (lambda (match) + (let ((tag (match-string 1 match)) + (attrs (string-trim-right (or (match-string 2 match) "")))) + (if (member (downcase tag) pulpie-void-tags) + (format "<%s%s>" tag attrs) + (format "<%s%s></%s>" tag attrs tag)))) + s)) + +(defun pulpie--serialize (node) + "Serialize NODE to an HTML string, dropping noise tags." + (with-temp-buffer + (dom-print (pulpie--prune node)) + (let ((s (pulpie--self-closing (buffer-string)))) + ;; <pre> bodies stay verbatim; everything else is collapsed + (if (string-match-p "<pre[ >]" s) s (pulpie--collapse s))))) + +(defun pulpie--merge-to-ancestors (nodes labels dom) + "HTML for the kept blocks, merged up into their containing elements. +Classifying leaves loses the structure above them, so each kept block is +lifted to the largest ancestor that contains only kept blocks - that keeps +ul, table and section wrappers intact without dragging in neighbouring blocks +classified as other. DOM nodes are plain lists, so `eq' on the node is a +stable identity." + (let ((total (make-hash-table :test #'eq)) + (kept (make-hash-table :test #'eq)) + (parents (make-hash-table :test #'eq))) + (pulpie--walk dom (lambda (n parent) (when parent (puthash n parent parents)))) + (cl-loop for node in nodes for label in labels do + (let ((anc node)) + (while anc + (cl-incf (gethash anc total 0)) + (cl-incf (gethash anc kept 0) label) + (setq anc (gethash anc parents))))) + (let ((roots (make-hash-table :test #'eq)) + (out '())) + (cl-loop for node in nodes for label in labels when (= label 1) do + (let ((best node) (anc (gethash node parents))) + (while (and anc (not (memq (dom-tag anc) '(html body))) + (= (gethash anc total 0) (gethash anc kept 0))) + (setq best anc anc (gethash anc parents))) + (puthash best t roots))) + (pulpie--walk dom + (lambda (n _parent) + (when (gethash n roots) + (remhash n roots) + (let ((html (pulpie--serialize n))) + (unless (string-empty-p html) + (setq out (append out (list html)))))))) + out))) + +;;; Entry point + +(defun pulpie-main-content (html) + "Return the main content of the document HTML as an HTML fragment. +See `pulpie-main-html' for the networked version; this one is for a document +you already have, e.g. an eww buffer's source." + (pcase-let ((`(,markup ,nodes ,dom) (pulpie--page-blocks html))) + (unless markup (error "No content blocks found")) + (mapconcat #'identity + (pulpie--merge-to-ancestors nodes (pulpie--classify markup) dom) + "\n"))) + +;;;###autoload +(defun pulpie-main-html (url) + "Fetch URL and return its main content as an HTML fragment. +The page's blocks are classified as main or other by the pulpie GGUF served at +`pulpie-server'; kept blocks are merged up into their containing elements, so +the fragment keeps headings, lists, tables and links as the page had them. +Signal an error if the page cannot be fetched or has no classifiable blocks." + (pulpie-main-content (pulpie--get url))) + +(provide 'pulpie)