commit 2d76af7f779fdf79a9a87bbc58b86789d79062a7
Author: Vineet Kumar <git@vineetk.net>
Date: Fri, 20 Nov 2026 01:11:19 +0100
initial payload
the script was made by qwen3.8-flash-next, with quite some steering by
me. I know it's ugly.
Diffstat:
| A | pulpie.el | | | 363 | +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ |
1 file changed, 363 insertions(+), 0 deletions(-)
diff --git a/pulpie.el b/pulpie.el
@@ -0,0 +1,363 @@
+;;; pulpie.el --- eww-readable replacement engine, using pulpie BERT as classifier -*- lexical-binding: t; -*-
+
+;; Version: 1.0
+;; Package-Requires: ((emacs "28.1"))
+
+;;; Commentary:
+
+;; One entry point:
+;;
+;; (pulpie-main-html "https://example.com/article")
+;; => "<main><h1>...</h1><p>...</p></main>"
+;;
+;; It returns the page's main content as an HTML fragment - navigation,
+;; sidebars, comments and footers dropped - intended as the reader step in
+;; front of eww, which otherwise shows the whole document.
+;;
+;; Classification runs on a llama-server serving a pulpie GGUF:
+;;
+;; llama-server -m pulpie-orange-small-cls-f16.gguf --embeddings \
+;; -b 8192 -ub 8192 -c 8192 --port 8888
+;;
+;; How it works: the page is split into leaf blocks, packed into the model's
+;; chunk template
+;;
+;; [BOS] block0 <|sep|> block1 <|sep|> ... <|sep|> [EOS]
+;;
+;; and every <|sep|> row of /embedding carries that block's [other, main]
+;; logits. Token ids come from the server's /tokenize endpoint, so they match
+;; the tokenizer baked into the GGUF exactly.
+;;
+;; Blocks are classified as leaves but emitted as whole subtrees: each kept
+;; block is lifted to the largest ancestor containing only kept blocks, so
+;; <ul>, <table> and section wrappers survive.
+
+;;; Code:
+
+(require 'cl-lib)
+(require 'dom)
+(require 'json)
+(require 'pcase)
+(require 'url)
+(require 'url-http)
+
+(defgroup pulpie nil
+ "Main-content extraction with a pulpie GGUF behind llama-server."
+ :group 'hypermedia)
+
+;; (defcustom pulpie-server "http://localhost:8888"
+;; "Base URL of the llama-server serving a pulpie GGUF."
+;; :type 'string :group 'pulpie)
+(defcustom pulpie-server "http://llm.demiurge.epistemia/upstream/pulpie-orange-small"
+ "Base URL of the llama-server serving a pulpie GGUF."
+ :type 'string :group 'pulpie)
+
+(defcustom pulpie-max-tokens 8192
+ "Token budget per request; must not exceed the server's -b/-ub/-c value."
+ :type 'integer :group 'pulpie)
+
+(defconst pulpie-bos "<|begin_of_text|>" "Chunk prefix.")
+(defconst pulpie-eos "<|end_of_text|>" "Chunk suffix.")
+(defconst pulpie-sep "<|sep|>" "Block separator.")
+(defconst pulpie-sep-token-id 128256 "Vocab id of <|sep|>.")
+
+(defconst pulpie-block-tags
+ '(p li h1 h2 h3 h4 h5 h6 pre blockquote figcaption td th dd dt summary)
+ "Block-level tags whose text is classified (leaf elements only).")
+
+(defconst pulpie-noise-tags
+ '(script style noscript iframe template svg canvas form)
+ "Tags dropped when merged blocks are serialized.")
+
+(defconst pulpie-void-tags
+ '("area" "base" "br" "col" "embed" "hr" "img" "input" "link" "meta"
+ "source" "track" "wbr")
+ "HTML elements that must not be printed with a closing tag.")
+
+;;; HTTP
+
+(defun pulpie--header-end ()
+ "Return the position just past the HTTP header block."
+ (save-excursion
+ (goto-char (point-min))
+ (if (re-search-forward "\r?\n\r?\n" nil t) (match-end 0) (point-min))))
+
+(defun pulpie--body (buffer url)
+ "Return BUFFER's decoded body as a string, then kill BUFFER.
+URL is only used in the error message. Signals on a 4xx/5xx status."
+ (with-current-buffer buffer
+ (unwind-protect
+ (let* ((status (save-excursion
+ (goto-char (point-min))
+ (when (looking-at "HTTP/[0-9.]+ \\([0-9]+\\)")
+ (string-to-number (match-string 1)))))
+ (header-end (pulpie--header-end))
+ (charset (save-excursion
+ (goto-char (point-min))
+ (when (re-search-forward
+ "^Content-Type:.*?charset=\\([^; \r\n]+\\)"
+ header-end t)
+ (match-string 1))))
+ ;; no charset is UTF-8, not the ISO-8859-1 of an old RFC
+ (coding (let ((sym (and charset (ignore-errors (intern charset)))))
+ (if (and sym (coding-system-p sym)) sym 'utf-8))))
+ (when (and status (>= status 400))
+ (error "Request to %s failed (%s): %s"
+ url status
+ (string-trim
+ (buffer-substring-no-properties header-end (point-max)))))
+ (let ((body (buffer-substring-no-properties (pulpie--header-end) (point-max))))
+ ;; url.el leaves the body as raw bytes (a unibyte string); decode
+ ;; them here. decode-coding-region on the buffer is a no-op.
+ (if (multibyte-string-p body)
+ body
+ (decode-coding-string body coding))))
+ (kill-buffer buffer))))
+
+(defun pulpie--get (url)
+ "Fetch URL and return its decoded body."
+ (let ((url-user-agent url-user-agent)
+ (url-mime-accept-string "text/html,application/xhtml+xml,*/*")
+ (url-show-status nil)
+ (url-request-method "GET")
+ (url-request-extra-headers
+ (list (cons "User-Agent" pulpie-user-agent)
+ (cons "Accept" "text/html,application/xhtml+xml,*/*"))))
+ (pulpie--body (url-retrieve-synchronously url t nil 60) url)))
+
+(defun pulpie--post-json (url payload)
+ "POST PAYLOAD as JSON to URL and return the parsed response."
+ (let* ((url-show-status nil)
+ (url-request-method "POST")
+ (url-request-extra-headers
+ (list (cons "Content-Type" "application/json")
+ (cons "User-Agent" pulpie-user-agent)))
+ (url-request-data (encode-coding-string (json-encode payload) 'utf-8)))
+ (json-parse-string (pulpie--body (url-retrieve-synchronously url t nil 300) url)
+ ;; keys come back interned, hence `assq' below
+ :object-type 'alist :array-type 'list)))
+
+(defun pulpie--tokenize (text)
+ "Return the token id list of TEXT, from the server's tokenizer."
+ (cdr (assq 'tokens (pulpie--post-json (concat pulpie-server "/tokenize")
+ (list (cons "content" text))))))
+
+(defun pulpie--logits (text)
+ "Return TEXT's per-token [other, main] logits as a list of rows."
+ (let ((first (car (pulpie--post-json (concat pulpie-server "/embedding")
+ (list (cons "content" text))))))
+ (unless first (error "Empty /embedding response"))
+ (cdr (assq 'embedding first))))
+
+;;; DOM -> blocks
+
+(defun pulpie--element-p (node)
+ "Non-nil if NODE is a DOM element rather than a text node."
+ (and (consp node) (symbolp (car node))))
+
+(defun pulpie--walk (node fn)
+ "Call FN with every element under NODE in document order, plus its parent."
+ (letrec ((go (lambda (n parent)
+ (funcall fn n parent)
+ (dolist (child (dom-children n))
+ (when (pulpie--element-p child)
+ (funcall go child n))))))
+ (when (pulpie--element-p node) (funcall go node nil))))
+
+(defun pulpie--has-block-descendant-p (node)
+ "Non-nil if any descendant of NODE is one of `pulpie-block-tags'."
+ (catch 'found
+ (dolist (child (dom-children node))
+ (when (pulpie--element-p child)
+ (when (memq (dom-tag child) pulpie-block-tags) (throw 'found t))
+ (when (pulpie--has-block-descendant-p child) (throw 'found t))))
+ nil))
+
+(defun pulpie--leaf-block-p (node)
+ "Non-nil if NODE is a block containing no other block."
+ (and (memq (dom-tag node) pulpie-block-tags)
+ (not (pulpie--has-block-descendant-p node))))
+
+(defun pulpie--escape (s)
+ "Escape S for the classifier markup.
+Mirrors Python's html.escape(s, quote=True), which is what the model was
+verified against - including the apostrophe, because an apostrophe becomes
+' and that changes the token ids the classifier sees."
+ (let ((s (replace-regexp-in-string "&" "&" s)))
+ (setq s (replace-regexp-in-string "<" "<" s))
+ (setq s (replace-regexp-in-string ">" ">" s))
+ (setq s (replace-regexp-in-string "\"" """ s))
+ (replace-regexp-in-string "'" "'" s)))
+
+(defun pulpie--collapse (s)
+ "Collapse every whitespace run in S to a single space."
+ (string-trim (replace-regexp-in-string "[ \t\r\n]+" " " s)))
+
+(defun pulpie--page-blocks (html)
+ "Split HTML into leaf blocks, returning (MARKUP NODES DOM).
+MARKUP is what the classifier sees: a div carrying `_item_id' and the block's
+raw text, mirroring pulpie's simplify output, since the model was distilled on
+that segmentation and not on the page's original tag names. NODES are the
+matching DOM elements, used to rebuild HTML around the kept blocks."
+ (let* ((dom (with-temp-buffer
+ (insert html)
+ (libxml-parse-html-region (point-min) (point-max))))
+ (root (or (car (dom-by-tag dom 'body)) dom))
+ (markup '()) (nodes '()))
+ (pulpie--walk root
+ (lambda (n _parent)
+ (when (pulpie--leaf-block-p n)
+ (let ((text (string-trim (or (dom-inner-text n) ""))))
+ (when (>= (length text) 2)
+ (push (format "<div _item_id=\"%d\">%s</div>"
+ (length markup) (pulpie--escape text))
+ markup)
+ (push n nodes))))))
+ (list (nreverse markup) (nreverse nodes) dom)))
+
+;;; Labels
+
+(defun pulpie--argmax (row)
+ "Return the index of the largest value in ROW."
+ (let ((best 0) (best-value (car row)) (i 0))
+ (dolist (v row)
+ (when (> v best-value) (setq best i best-value v))
+ (setq i (1+ i)))
+ best))
+
+(defun pulpie--label-chunk (chunk)
+ "Classify CHUNK (a list of block markup strings), returning 0/1 per block."
+ (let* ((text (concat pulpie-bos
+ (mapconcat (lambda (b) (concat b pulpie-sep)) chunk "")
+ pulpie-eos))
+ (ids (pulpie--tokenize text))
+ (rows (pulpie--logits text))
+ (sep-rows '()) (i 0))
+ (dolist (tok ids)
+ (when (= tok pulpie-sep-token-id) (push i sep-rows))
+ (setq i (1+ i)))
+ (setq sep-rows (nreverse sep-rows))
+ (unless (= (length sep-rows) (length chunk))
+ (error "%d <|sep|> rows for %d blocks" (length sep-rows) (length chunk)))
+ (mapcar (lambda (row-index) (pulpie--argmax (nth row-index rows))) sep-rows)))
+
+(defun pulpie--block-sizes (blocks)
+ "Return each block's token count, from a single /tokenize round trip."
+ (let ((ids (pulpie--tokenize (mapconcat #'identity blocks pulpie-sep)))
+ (sizes '()) (start 0) (i 0))
+ (dolist (tok ids)
+ (when (= tok pulpie-sep-token-id)
+ (push (- i start) sizes)
+ (setq start (1+ i)))
+ (setq i (1+ i)))
+ (nreverse (cons (- (length ids) start) sizes))))
+
+(defun pulpie--classify (blocks &optional max-tokens)
+ "Label every block 1 (main) or 0 (other), one request per chunk."
+ (let* ((max-tokens (or max-tokens pulpie-max-tokens))
+ (sizes (pulpie--block-sizes blocks))
+ (labels '()) (chunk '()) (chunk-tokens 0))
+ (cl-loop for block in blocks for size in sizes do
+ ;; tokens so far + this block + one <|sep|> per block + BOS/EOS
+ (when (and chunk (> (+ chunk-tokens size (length chunk) 3) max-tokens))
+ (setq labels (nconc labels (pulpie--label-chunk chunk)))
+ (setq chunk '() chunk-tokens 0))
+ (setq chunk (append chunk (list block))
+ chunk-tokens (+ chunk-tokens size)))
+ (when chunk (setq labels (nconc labels (pulpie--label-chunk chunk))))
+ labels))
+
+;;; Rebuild
+
+(defun pulpie--prune (node)
+ "Return a copy of NODE without `pulpie-noise-tags' descendants."
+ (if (stringp node)
+ node
+ (cons (dom-tag node)
+ (cons (dom-attributes node)
+ (let (acc)
+ (dolist (child (dom-children node))
+ (cond ((stringp child) (push child acc))
+ ((not (memq (dom-tag child) pulpie-noise-tags))
+ (push (pulpie--prune child) acc))))
+ (nreverse acc))))))
+
+(defun pulpie--self-closing (s)
+ "Fix up libxml's XHTML-style self-closing tags in S.
+libxml prints every empty element as `<foo />', which HTML reads as an
+ordinary start tag - unbalanced inside a fragment. Void elements lose the
+slash; everything else gets an explicit close."
+ (replace-regexp-in-string
+ "<\\([a-zA-Z][a-zA-Z0-9-]*\\)\\( [^>]*\\)?/>"
+ (lambda (match)
+ (let ((tag (match-string 1 match))
+ (attrs (string-trim-right (or (match-string 2 match) ""))))
+ (if (member (downcase tag) pulpie-void-tags)
+ (format "<%s%s>" tag attrs)
+ (format "<%s%s></%s>" tag attrs tag))))
+ s))
+
+(defun pulpie--serialize (node)
+ "Serialize NODE to an HTML string, dropping noise tags."
+ (with-temp-buffer
+ (dom-print (pulpie--prune node))
+ (let ((s (pulpie--self-closing (buffer-string))))
+ ;; <pre> bodies stay verbatim; everything else is collapsed
+ (if (string-match-p "<pre[ >]" s) s (pulpie--collapse s)))))
+
+(defun pulpie--merge-to-ancestors (nodes labels dom)
+ "HTML for the kept blocks, merged up into their containing elements.
+Classifying leaves loses the structure above them, so each kept block is
+lifted to the largest ancestor that contains only kept blocks - that keeps
+ul, table and section wrappers intact without dragging in neighbouring blocks
+classified as other. DOM nodes are plain lists, so `eq' on the node is a
+stable identity."
+ (let ((total (make-hash-table :test #'eq))
+ (kept (make-hash-table :test #'eq))
+ (parents (make-hash-table :test #'eq)))
+ (pulpie--walk dom (lambda (n parent) (when parent (puthash n parent parents))))
+ (cl-loop for node in nodes for label in labels do
+ (let ((anc node))
+ (while anc
+ (cl-incf (gethash anc total 0))
+ (cl-incf (gethash anc kept 0) label)
+ (setq anc (gethash anc parents)))))
+ (let ((roots (make-hash-table :test #'eq))
+ (out '()))
+ (cl-loop for node in nodes for label in labels when (= label 1) do
+ (let ((best node) (anc (gethash node parents)))
+ (while (and anc (not (memq (dom-tag anc) '(html body)))
+ (= (gethash anc total 0) (gethash anc kept 0)))
+ (setq best anc anc (gethash anc parents)))
+ (puthash best t roots)))
+ (pulpie--walk dom
+ (lambda (n _parent)
+ (when (gethash n roots)
+ (remhash n roots)
+ (let ((html (pulpie--serialize n)))
+ (unless (string-empty-p html)
+ (setq out (append out (list html))))))))
+ out)))
+
+;;; Entry point
+
+(defun pulpie-main-content (html)
+ "Return the main content of the document HTML as an HTML fragment.
+See `pulpie-main-html' for the networked version; this one is for a document
+you already have, e.g. an eww buffer's source."
+ (pcase-let ((`(,markup ,nodes ,dom) (pulpie--page-blocks html)))
+ (unless markup (error "No content blocks found"))
+ (mapconcat #'identity
+ (pulpie--merge-to-ancestors nodes (pulpie--classify markup) dom)
+ "\n")))
+
+;;;###autoload
+(defun pulpie-main-html (url)
+ "Fetch URL and return its main content as an HTML fragment.
+The page's blocks are classified as main or other by the pulpie GGUF served at
+`pulpie-server'; kept blocks are merged up into their containing elements, so
+the fragment keeps headings, lists, tables and links as the page had them.
+Signal an error if the page cannot be fetched or has no classifiable blocks."
+ (pulpie-main-content (pulpie--get url)))
+
+(provide 'pulpie)