pulpie-eww.el

replacing eww-readable with pulpie-orange-small
Log | Files | Refs | LICENSE

pulpie-eww.el (12366B)


      1 ;;; pulpie-eww.el --- eww-readable replacement engine, using pulpie BERT as classifier  -*- lexical-binding: t; -*-
      2 
      3 ;; Version: 1.0
      4 ;; Package-Requires: ((emacs "30.1"))
      5 
      6 ;;; Commentary:
      7 
      8 ;; Enable with (pulpie-eww-mode 1): wraps `eww-readable-dom' with
      9 ;; `pulpie-readable-dom', so eww's reader classifies main content with a
     10 ;; pulpie classifier instead of heuristics.  The DOM eww already parsed is
     11 ;; used as-is; the only requests pulpie makes go to `pulpie-server', a
     12 ;; llama-server (or compatible) exposing the model, e.g.:
     13 ;;
     14 ;;   llama-server -m pulpie-orange-small-cls-f16.gguf --embeddings \
     15 ;;       -b 8192 -ub 8192 -c 8192
     16 ;;
     17 ;; The DOM is split into leaf blocks, packed into the model's chunk template
     18 ;;
     19 ;;   [BOS] block0 <|sep|> block1 <|sep|> ... <|sep|> [EOS]
     20 ;;
     21 ;; each <|sep|> row of /embedding carrying that block's [other, main]
     22 ;; logits.  Token ids come from /tokenize so they match the model's own
     23 ;; tokenizer.
     24 ;;
     25 ;; Blocks are classified as leaves but emitted as whole subtrees: each kept
     26 ;; block is lifted to the largest ancestor containing only kept blocks, so
     27 ;; <ul>, <table> and section wrappers survive.
     28 ;;
     29 ;; In terms of performance overhead, it's around 0.1-0.8s when
     30 ;; enabling reader mode on my machines. That's with the classifier
     31 ;; running on CPU and not GPU.
     32 
     33 ;;; Code:
     34 
     35 (require 'cl-lib)
     36 (require 'dom)
     37 (require 'json)
     38 (require 'pcase)
     39 (require 'url)
     40 
     41 (defgroup pulpie nil
     42   "Main-content extraction with a pulpie classifier."
     43   :group 'hypermedia)
     44 
     45 (defcustom pulpie-server "http://localhost:9931"
     46   "Base URL of the llama-server exposing the pulpie classifier."
     47   :type 'string :group 'pulpie)
     48 
     49 (defcustom pulpie-max-tokens 8192
     50   "Token budget per request; must not exceed the server's context size."
     51   :type 'integer :group 'pulpie)
     52 
     53 (defconst pulpie-bos "<|begin_of_text|>" "Chunk prefix.")
     54 (defconst pulpie-eos "<|end_of_text|>" "Chunk suffix.")
     55 (defconst pulpie-sep "<|sep|>" "Block separator.")
     56 (defconst pulpie-sep-token-id 128256 "Vocab id of <|sep|>.")
     57 
     58 (defconst pulpie-block-tags
     59   '(p li h1 h2 h3 h4 h5 h6 pre blockquote figcaption td th dd dt summary)
     60   "Block-level tags whose text is classified (leaf elements only).")
     61 
     62 ;;; HTTP
     63 
     64 (defun pulpie--header-end ()
     65   "Return the position just past the HTTP header block."
     66   (save-excursion
     67     (goto-char (point-min))
     68     (if (re-search-forward "\r?\n\r?\n" nil t) (match-end 0) (point-min))))
     69 
     70 (defun pulpie--body (buffer url)
     71   "Return BUFFER's body decoded as UTF-8, then kill BUFFER.
     72 URL is only used in the error message.  Signals on a 4xx/5xx status."
     73   (with-current-buffer buffer
     74     (unwind-protect
     75         (let ((header-end (pulpie--header-end)))
     76           (goto-char (point-min))
     77           (when (and (looking-at "HTTP/[0-9.]+ \([0-9]+\)")
     78                      (>= (string-to-number (match-string 1)) 400))
     79             (error "Request to %s failed (%s): %s" url (match-string 1)
     80                    (string-trim
     81                     (buffer-substring-no-properties header-end (point-max)))))
     82           (let ((body (buffer-substring-no-properties header-end (point-max))))
     83             ;; url.el may leave the body as raw bytes
     84             (if (multibyte-string-p body)
     85                 body
     86               (decode-coding-string body 'utf-8))))
     87       (kill-buffer buffer))))
     88 
     89 (defun pulpie--post-json (url payload)
     90   "POST PAYLOAD as JSON to URL and return the parsed response."
     91   (let* ((url-show-status nil)
     92          (url-request-method "POST")
     93          (url-request-extra-headers
     94           (list (cons "Content-Type" "application/json")))
     95          (url-request-data (encode-coding-string (json-encode payload) 'utf-8)))
     96     (json-parse-string (pulpie--body (url-retrieve-synchronously url t nil 300) url)
     97                        ;; keys come back interned, hence `assq' below
     98                        :object-type 'alist :array-type 'list)))
     99 
    100 (defun pulpie--tokenize (text)
    101   "Return the token id list of TEXT, from the server's tokenizer."
    102   (cdr (assq 'tokens (pulpie--post-json (concat pulpie-server "/tokenize")
    103                                         (list (cons "content" text))))))
    104 
    105 (defun pulpie--logits (text)
    106   "Return TEXT's per-token [other, main] logits as a list of rows."
    107   (let ((first (car (pulpie--post-json (concat pulpie-server "/embedding")
    108                                        (list (cons "content" text))))))
    109     (unless first (error "Empty /embedding response"))
    110     (cdr (assq 'embedding first))))
    111 
    112 ;;; DOM -> blocks
    113 
    114 (defun pulpie--element-p (node)
    115   "Non-nil if NODE is a DOM element rather than a text node."
    116   (and (consp node) (symbolp (car node))))
    117 
    118 (defun pulpie--walk (node fn)
    119   "Call FN with every element under NODE in document order, plus its parent."
    120   (letrec ((go (lambda (n parent)
    121                  (funcall fn n parent)
    122                  (dolist (child (dom-children n))
    123                    (when (pulpie--element-p child)
    124                      (funcall go child n))))))
    125     (when (pulpie--element-p node) (funcall go node nil))))
    126 
    127 (defun pulpie--has-block-descendant-p (node)
    128   "Non-nil if any descendant of NODE is one of `pulpie-block-tags'."
    129   (catch 'found
    130     (dolist (child (dom-children node))
    131       (when (pulpie--element-p child)
    132         (when (memq (dom-tag child) pulpie-block-tags) (throw 'found t))
    133         (when (pulpie--has-block-descendant-p child) (throw 'found t))))
    134     nil))
    135 
    136 (defun pulpie--leaf-block-p (node)
    137   "Non-nil if NODE is a block containing no other block."
    138   (and (memq (dom-tag node) pulpie-block-tags)
    139        (not (pulpie--has-block-descendant-p node))))
    140 
    141 (defun pulpie--escape (s)
    142   "Escape S for the classifier markup.
    143 The model was trained on markup where the apostrophe becomes &#x27;, so
    144 this set of escapes, and no other, must be preserved: different escaping
    145 yields different token ids."
    146   (let ((s (replace-regexp-in-string "&" "&amp;" s)))
    147     (setq s (replace-regexp-in-string "<" "&lt;" s))
    148     (setq s (replace-regexp-in-string ">" "&gt;" s))
    149     (setq s (replace-regexp-in-string "\"" "&quot;" s))
    150     (replace-regexp-in-string "'" "&#x27;" s)))
    151 
    152 (defun pulpie--blocks-from-dom (dom)
    153   "Split DOM into leaf blocks, returning (MARKUP NODES).
    154 MARKUP is what the classifier sees: a div carrying `_item_id' and the
    155 block's raw text - the model was trained on this segmentation, not on the
    156 page's original tag names.  NODES are the matching DOM elements."
    157   (let ((root (or (car (dom-by-tag dom 'body)) dom))
    158         (markup '()) (nodes '()))
    159     (pulpie--walk root
    160                   (lambda (n _parent)
    161                     (when (pulpie--leaf-block-p n)
    162                       (let ((text (string-trim (or (dom-inner-text n) ""))))
    163                         (when (>= (length text) 2)
    164                           (push (format "<div _item_id=\"%d\">%s</div>"
    165                                         (length markup) (pulpie--escape text))
    166                                 markup)
    167                           (push n nodes))))))
    168     (list (nreverse markup) (nreverse nodes))))
    169 
    170 ;;; Labels
    171 
    172 (defun pulpie--argmax (row)
    173   "Return the index of the largest value in ROW."
    174   (let ((best 0) (best-value (car row)) (i 0))
    175     (dolist (v row)
    176       (when (> v best-value) (setq best i best-value v))
    177       (setq i (1+ i)))
    178     best))
    179 
    180 (defun pulpie--label-chunk (chunk)
    181   "Classify CHUNK (a list of block markup strings), returning 0/1 per block."
    182   (let* ((text (concat pulpie-bos
    183                        (mapconcat (lambda (b) (concat b pulpie-sep)) chunk "")
    184                        pulpie-eos))
    185          (ids (pulpie--tokenize text))
    186          (rows (pulpie--logits text))
    187          (sep-rows '()) (i 0))
    188     (dolist (tok ids)
    189       (when (= tok pulpie-sep-token-id) (push i sep-rows))
    190       (setq i (1+ i)))
    191     (setq sep-rows (nreverse sep-rows))
    192     (unless (= (length sep-rows) (length chunk))
    193       (error "%d <|sep|> rows for %d blocks" (length sep-rows) (length chunk)))
    194     (mapcar (lambda (row-index) (pulpie--argmax (nth row-index rows))) sep-rows)))
    195 
    196 (defun pulpie--block-sizes (blocks)
    197   "Return each block's token count, from a single /tokenize round trip."
    198   (let ((ids (pulpie--tokenize (mapconcat #'identity blocks pulpie-sep)))
    199         (sizes '()) (start 0) (i 0))
    200     (dolist (tok ids)
    201       (when (= tok pulpie-sep-token-id)
    202         (push (- i start) sizes)
    203         (setq start (1+ i)))
    204       (setq i (1+ i)))
    205     (nreverse (cons (- (length ids) start) sizes))))
    206 
    207 (defun pulpie--classify (blocks)
    208   "Label every block 1 (main) or 0 (other), one request per chunk."
    209   (let* ((sizes (pulpie--block-sizes blocks))
    210          (labels '()) (chunk '()) (chunk-tokens 0))
    211     (cl-loop for block in blocks for size in sizes do
    212              ;; tokens so far + this block + one <|sep|> per block + BOS/EOS
    213              (when (and chunk (> (+ chunk-tokens size (length chunk) 3)
    214                                  pulpie-max-tokens))
    215                (setq labels (nconc labels (pulpie--label-chunk chunk)))
    216                (setq chunk '() chunk-tokens 0))
    217              (setq chunk (append chunk (list block))
    218                    chunk-tokens (+ chunk-tokens size)))
    219     (when chunk (setq labels (nconc labels (pulpie--label-chunk chunk))))
    220     labels))
    221 
    222 ;;; Rebuild
    223 
    224 (defun pulpie--kept-roots (nodes labels dom)
    225   "Return the main-content roots for NODES labelled 1 by LABELS, in document order.
    226 Classifying leaves loses the structure above them, so each kept block is
    227 lifted to the largest ancestor that contains only kept blocks - that keeps
    228 ul, table and section wrappers intact without dragging in neighbouring blocks
    229 classified as other.  DOM nodes are plain lists, so `eq' on the node is a
    230 stable identity."
    231   (let ((total (make-hash-table :test #'eq))
    232         (kept (make-hash-table :test #'eq))
    233         (parents (make-hash-table :test #'eq)))
    234     (pulpie--walk dom (lambda (n parent) (when parent (puthash n parent parents))))
    235     (cl-loop for node in nodes for label in labels do
    236              (let ((anc node))
    237                (while anc
    238                  (cl-incf (gethash anc total 0))
    239                  (cl-incf (gethash anc kept 0) label)
    240                  (setq anc (gethash anc parents)))))
    241     (let ((roots (make-hash-table :test #'eq))
    242           (out '()))
    243       (cl-loop for node in nodes for label in labels when (= label 1) do
    244                (let ((best node) (anc (gethash node parents)))
    245                  (while (and anc (not (memq (dom-tag anc) '(html body)))
    246                              (= (gethash anc total 0) (gethash anc kept 0)))
    247                    (setq best anc anc (gethash anc parents)))
    248                  (puthash best t roots)))
    249       (pulpie--walk dom
    250                     (lambda (n _parent)
    251                       (when (gethash n roots)
    252                         (remhash n roots)
    253                         (push n out))))
    254       (nreverse out))))
    255 
    256 ;;; Entry points
    257 
    258 (defun pulpie-readable-dom (dom)
    259   "Drop-in replacement for `eww-readable-dom': return DOM as a `body'
    260 document containing only the main content, or nil if it has no
    261 classifiable blocks.  Only `pulpie-server' is contacted."
    262   (pcase-let ((`(,markup ,nodes) (pulpie--blocks-from-dom dom)))
    263     (when markup
    264       `(body nil ,@(pulpie--kept-roots nodes (pulpie--classify markup) dom)))))
    265 
    266 (defun pulpie--eww-readable-dom (orig dom)
    267   "Around-advice for `eww-readable-dom': try pulpie on DOM, else ORIG.
    268 Fall back to the built-in heuristics if the pulpie server is unreachable
    269 or the page has no classifiable blocks."
    270   (message "pulpie: DOM received from eww, classifying...")
    271   (let ((start (float-time))
    272         (result (condition-case err
    273                     (or (pulpie-readable-dom dom) (funcall orig dom))
    274                   (error
    275                    (message "pulpie: %s (using built-in readability)"
    276                             (error-message-string err))
    277                    (funcall orig dom)))))
    278     (message "pulpie: returning DOM to eww (%.1fs)"
    279              (- (float-time) start))
    280     result))
    281 
    282 ;;;###autoload
    283 (define-minor-mode pulpie-eww-mode
    284   "Read the main content of pages in eww with pulpie instead of heuristics.
    285 Wraps `eww-readable-dom' with `pulpie-readable-dom', so `eww-readable' and
    286 auto-readable pages (`eww-readable-urls') classify via `pulpie-server'."
    287   :global t :group 'pulpie
    288   (if pulpie-eww-mode
    289       (advice-add 'eww-readable-dom :around #'pulpie--eww-readable-dom)
    290     (advice-remove 'eww-readable-dom #'pulpie--eww-readable-dom)))
    291 
    292 (provide 'pulpie-eww)