Add JSONP support on other endpoints

Refactored the code to define and process endpoints rather than doing
it imperatively, which avoided a lot of duplicated code.

breezewiki.rkt

@@ -27,9 +27,7 @@
 (require-reloadable "src/page-static-archive.rkt" page-static-archive)
 (require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher)
 (require-reloadable "src/page-wiki.rkt" page-wiki)
-(require-reloadable "src/page-wiki.rkt" page-wiki-with-data)
 (require-reloadable "src/page-wiki-offline.rkt" page-wiki-offline)
-(require-reloadable "src/page-wiki-jsonp.rkt" page-wiki-jsonp)
 (require-reloadable "src/page-file.rkt" page-file)
 
 (reload!)
@@ -59,9 +57,7 @@
       page-set-user-settings
       page-static-archive
       page-wiki
-      page-wiki-with-data
       page-wiki-offline
-      page-wiki-jsonp
       page-file
       redirect-wiki-home
       static-dispatcher

dist.rkt

@@ -16,9 +16,8 @@
 (require (only-in "src/page-static.rkt" static-dispatcher))
 (require (only-in "src/page-static-archive.rkt" page-static-archive))
 (require (only-in "src/page-subdomain.rkt" subdomain-dispatcher))
-(require (only-in "src/page-wiki.rkt" page-wiki page-wiki-with-data))
+(require (only-in "src/page-wiki.rkt" page-wiki))
 (require (only-in "src/page-wiki-offline.rkt" page-wiki-offline))
-(require (only-in "src/page-wiki-jsonp.rkt" page-wiki-jsonp))
 (require (only-in "src/page-file.rkt" page-file))
 
 (serve/launch/wait
@@ -43,8 +42,6 @@
     page-static-archive
     page-wiki
     page-wiki-offline
-    page-wiki-with-data
-    page-wiki-jsonp
     page-file
     redirect-wiki-home
     static-dispatcher

lib/tree-updater.rkt

@@ -12,7 +12,7 @@
  update-tree-wiki)
 
 (define (preprocess-html-wiki html)
-  (regexp-replace* #rx"(<(?:td|figcaption)[^>]*?>\n?)(?:<li>|[ \t]*?<p class=\"caption\">(.*?)</p>)"
+  (regexp-replace* #rx"(<(?:td|figcaption)[^>]*?>\n?)(?:[ \t]*<a href=\"[^\"]*\" class=\"info-icon\"><svg><use xlink:href=\"#wds-icons-info-small\"></use></svg></a>)?(?:<li>|[ \t]*?<p class=\"caption\">(.*?)</p>)"
                    html (λ (whole first-tag [contents #f])
                           (if (eq? (string-ref whole 1) #\f) ;; figcaption
                               (string-append first-tag "<span class=\"caption\">" contents "</span>")

src/dispatcher-tree.rkt

@@ -55,6 +55,7 @@
   (define tree
     (sequencer:make
      subdomain-dispatcher
+     (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
      (pathprocedure:make "/" (page ds page-home))
      (filter:make #rx"^/static/" (hash-ref ds 'static-dispatcher))
      (filter:make (pregexp "^/captcha/img/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-image)))
@@ -65,16 +66,11 @@
      (pathprocedure:make "/proxy" (page ds page-proxy))
      (pathprocedure:make "/search" (page ds page-global-search))
      (pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
-     (pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
-     (pathprocedure:make "/api/render/wiki" (page ds page-wiki-with-data))
      (filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
      (filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
      (if (config-true? 'feature_offline::enabled)
          (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
          (λ (_conn _req) (next-dispatcher)))
-     (if (config-true? 'feature_jsonp::enabled)
-         (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-jsonp)))
-         (λ (_conn _req) (next-dispatcher)))
      (filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
      (filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
      (filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))

src/endpoints.rkt

@@ -0,0 +1,248 @@
+#lang racket/base
+(require racket/format
+         racket/match
+         racket/string
+         ; libs
+         (prefix-in easy: net/http-easy)
+         json
+         ; html libs
+         html-writing
+         ; web server libs
+         web-server/http
+         web-server/dispatchers/dispatch
+         ; my libs
+         "application-globals.rkt"
+         "config.rkt"
+         "data.rkt"
+         "fandom-request.rkt"
+         "static-data.rkt"
+         "../lib/archive-file-mappings.rkt"
+         "../lib/thread-utils.rkt"
+         "../lib/url-utils.rkt"
+         "../lib/xexpr-utils.rkt")
+(require (for-syntax racket/base syntax/parse))
+
+(provide
+ define-endpoint
+ define-standard-handler
+ define-post-data-handler
+ define-jsonp-handler
+ make-switch-handler
+ (all-from-out racket/format
+               racket/match
+               json
+               net/http-easy
+               web-server/dispatchers/dispatch
+               "../lib/thread-utils.rkt"
+               "../lib/archive-file-mappings.rkt"
+               "config.rkt"
+               "data.rkt"
+               "fandom-request.rkt"
+               "static-data.rkt"))
+
+
+(define-for-syntax (fix-scopes here orig stx)
+  (define remover (make-syntax-delta-introducer here #f))
+  (define introducer (make-syntax-delta-introducer orig #f))
+  (introducer (remover stx 'remove)))
+
+
+(define-syntax (define-endpoint stx)
+  (syntax-parse stx
+    [(_ name
+        (~and (~seq all ...)
+              (~seq
+               ((~datum variables) variable ...)
+               ((~datum endpoints) endpoint ...)
+               ((~datum render) render ...))))
+     #'(define-syntax name #'(all ...))]))
+
+
+(define ((make-switch-handler #:standard standard #:jsonp jsonp #:post post) req)
+  (cond
+    [(equal? (request-method req) #"POST")
+     (post req)]
+    [(config-true? 'feature_jsonp::enabled)
+     (jsonp req)]
+    [else
+     (standard req)]))
+
+
+(define-syntax (define-standard-handler stx)
+  (syntax-parse stx
+    [(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...)
+     #:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref)))
+     (syntax-parse #'endpoint-syntax
+       #:context 'endpoint-data
+       [(((~datum variables) variable ...)
+         ((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...)
+         ((~datum render) render ...))
+        #:with endpoint-getters (for/list ([id (syntax-e #'(endpoint-id ...))]
+                                           [params (syntax-e #'((endpoint-param ...) ...))])
+                                  (with-syntax ([params (append (syntax-e params) #;'(("callback" . "proxy.wikipage")))])
+                                    (if (eq? (syntax-e id) 'siteinfo)
+                                        #'(λ ()
+                                            (siteinfo-fetch wikiname))
+                                        #'(λ ()
+                                            (fandom-get-api
+                                             wikiname
+                                             `params
+                                             #:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))))))
+        #:with endpoint-intermediates->values (for/list ([id (syntax-e #'(endpoint-id ...))])
+                                                (with-syntax ([id id])
+                                                  (if (eq? (syntax-e #'id) 'siteinfo)
+                                                      #'id
+                                                      #'(easy:response-json id))))
+
+        (fix-scopes
+         #'here stx
+         #'(define name-args
+             (response-handler
+              (let/cc k
+                variable ...
+                (define user-cookies (user-cookies-getter req))
+                (define-values (endpoint-id ...)
+                  (let-values ([(endpoint-id ...) (thread-values (~@ . endpoint-getters))])
+                    (for ([response (list endpoint-id ...)]
+                          #:when (easy:response? response))
+                      (match (easy:response-status-code response)
+                        [404
+                         (next-dispatcher)]
+                        [(or 403 406)
+                         (define body
+                           (generate-wiki-page
+                            `(div
+                              (p "Sorry! Fandom isn't allowing BreezeWiki to show pages right now.")
+                              (p "We'll automatically try again in 30 seconds, so please stay on this page and be patient.")
+                              (p (small "In a hurry? " (a (@ (href ,source-url)) "Click here to read the page on Fandom."))))
+                            #:req req
+                            #:source-url source-url
+                            #:wikiname wikiname
+                            #:title title
+                            #:siteinfo siteinfo))
+                         (k (response/output
+                             #:code 503
+                             #:headers (build-headers
+                                        always-headers
+                                        (header #"Retry-After" #"30")
+                                        (header #"Cache-Control" #"max-age=30, public")
+                                        (header #"Refresh" #"35"))
+                             (λ (out)
+                               (write-html body out))))]
+                        [(not 200)
+                         (k (error 'page-wiki "Tried to load page ~a/~a~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
+                                   wikiname
+                                   path
+                                   (easy:response-status-code response)
+                                   (easy:response-body response)))]
+                        [_ (void)]))
+                    (values (~@ . endpoint-intermediates->values))))
+                fn-body ...
+                render ...))))])]))
+
+
+(define-syntax (define-post-data-handler stx)
+  (syntax-parse stx
+    [(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...)
+     #:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref)))
+     (syntax-parse #'endpoint-syntax
+       #:context 'endpoint-data
+       [(((~datum variables) variable ...)
+         ((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...)
+         ((~datum render) render ...))
+        #:with endpoint-getters (for/list ([id (syntax-e #'(endpoint-id ...))]
+                                           [params (syntax-e #'((endpoint-param ...) ...))])
+                                  (with-syntax ([id id])
+                                    (if (eq? (syntax-e #'id) 'siteinfo)
+                                        #'(data->siteinfo (hash-ref post-data 'id))
+                                        #'(hash-ref post-data 'id))))
+
+        (fix-scopes
+         #'here stx
+         #'(define name-args
+             (response-handler
+              (let/cc k
+                (define (k-validation-error message)
+                  (k (response/jsexpr
+                      #:code 400
+                      #:headers always-headers
+                      `#hasheq((error .
+                                      #hasheq((code . "breezewiki")
+                                              (info . ,message)))))))
+                (define post-data/bytes (request-post-data/raw req))
+                (when (not post-data/bytes)
+                  (k-validation-error "POST requests only, please."))
+
+                (define origin-header
+                  (or (headers-assq* #"origin" (request-headers/raw req))
+                      (headers-assq* #"referer" (request-headers/raw req))))
+                (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin))))
+                  (k-validation-error "Origin/Referer header failed validation - cross-origin requests are not allowed here"))
+
+                (define post-data/string (bytes->string/utf-8 post-data/bytes))
+                (define post-data (string->jsexpr post-data/string))
+                (define-values (endpoint-id ...) (values (~@ . endpoint-getters)))
+                (define wikiname (hash-ref post-data 'wikiname))
+                (define path (hash-ref post-data 'path))
+                (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
+                fn-body ...
+                render ...))))])]))
+
+
+(define-syntax (define-jsonp-handler stx)
+  (syntax-parse stx
+    [(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...)
+     #:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref)))
+     (syntax-parse #'endpoint-syntax
+       #:context 'endpoint-data
+       [(((~datum variables) variable ...)
+         ((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...)
+         ((~datum render) render ...))
+        #:with endpoint-scripts (for/list ([id (syntax-e #'(endpoint-id ...))]
+                                           [params (syntax-e #'((endpoint-param ...) ...))])
+                                  (with-syntax ([id id]
+                                                [params (append (syntax-e params)
+                                                                `(("callback" . ,(format "proxy.~a" (syntax-e id)))))])
+                                    #'(script (@ (async)
+                                                 (src ,(format "https://~a.fandom.com/api.php?~a"
+                                                               wikiname
+                                                               (params->query `params)))
+                                                 (data-jsonp-var ,(format "~a" 'id))
+                                                 (onerror ,(format "proxy.~a({error: {code: 'disconnected', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})" 'id))))))
+
+        (fix-scopes
+         #'here stx
+         #'(define name-args
+             (response-handler
+              (let/cc k
+                variable ...
+                fn-body ...
+
+                (define body
+                  (generate-wiki-page
+                   `(div
+                     (noscript "You have to enable JavaScript to load wiki pages. Sorry!")
+                     (div (@ (id "loading")))
+                     (div (@ (id "progress-bar") (style "margin-bottom: 50vh"))
+                          (progress))
+                     (script #<<END
+var jsonpData = {}
+var proxy = new Proxy(jsonpData, {get(obj, prop) { return value => obj[prop] = value }})
+END
+                             )
+                     endpoint-scripts
+                     (script (@ (type "module") (src ,(get-static-url "jsonp.js")))))
+                   #:req req
+                   #:source-url source-url
+                   #:wikiname wikiname
+                   #:title title
+                   #:siteinfo siteinfo-default
+                   #:path path
+                   #:jsonp #t))
+                (when (config-true? 'debug)
+                  (xexp->html body))
+                (response/output
+                 #:code 200
+                 #:headers always-headers
+                 (λ (out)
+                   (write-html body out)))))))])]))

src/page-category.rkt

@@ -1,8 +1,5 @@
 #lang racket/base
-(require racket/dict
+(require racket/string
-         racket/list
-         racket/match
-         racket/string
          (prefix-in easy: net/http-easy)
          ; html libs
          html-parsing
@@ -11,14 +8,9 @@
          net/url
          web-server/http
          (only-in web-server/dispatchers/dispatch next-dispatcher)
-         #;(only-in web-server/http/redirect redirect-to)
          "application-globals.rkt"
-         "config.rkt"
+         "endpoints.rkt"
-         "data.rkt"
+         "../lib/tree-updater.rkt"
-         "fandom-request.rkt"
-         "page-wiki.rkt"
-         "../lib/syntax.rkt"
-         "../lib/thread-utils.rkt"
          "../lib/url-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
@@ -63,47 +55,47 @@
                         ,title)))
                  members))))))
 
-(define (page-category req)
-  (response-handler
-   (define wikiname (path/param-path (first (url-path (request-uri req)))))
-   (define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
-   (define origin (format "https://~a.fandom.com" wikiname))
-   (define source-url (format "~a/wiki/~a" origin prefixed-category))
 
-   (define-values (members-data page-data siteinfo)
+(define-endpoint
-     (thread-values
+  category-endpoint
-      (λ ()
+  [variables
-        (easy:response-json
+    (define wikiname (path/param-path (car (url-path (request-uri req)))))
-         (fandom-get-api
+    (define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
-          wikiname
+    (define segments (map path/param-path (cdr (url-path (request-uri req)))))
-          `(("action" . "query")
+    (define title (url-segments->guess-title segments))
-            ("list" . "categorymembers")
+    (define path (string-join (cdr segments) "/"))
-            ("cmtitle" . ,prefixed-category)
+    (define origin (format "https://~a.fandom.com" wikiname))
-            ("cmlimit" . "max")
+    (define source-url (format "~a/wiki/~a" origin prefixed-category))]
-            ("formatversion" . "2")
+  [endpoints
-            ("format" . "json")))))
+   (membersdata
-      (λ ()
+    (("action" . "query")
-        (easy:response-json
+      ("list" . "categorymembers")
-         (fandom-get-api
+      ("cmtitle" . ,prefixed-category)
-          wikiname
+      ("cmlimit" . "max")
-          `(("action" . "parse")
+      ("formatversion" . "2")
-            ("page" . ,prefixed-category)
+      ("format" . "json")))
-            ("prop" . "text|headhtml|langlinks")
+   (pagedata
-            ("formatversion" . "2")
+    (("action" . "parse")
-            ("format" . "json")))))
+      ("page" . ,prefixed-category)
-      (λ ()
+      ("prop" . "text|headhtml|langlinks")
-        (siteinfo-fetch wikiname))))
+      ("formatversion" . "2")
-
+      ("format" . "json")))
-   (define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category)))
+   (siteinfo
-   (define page-html (preprocess-html-wiki (jp "/parse/text" page-data "")))
+    (("action" . "query")
+      ("meta" . "siteinfo")
+      ("siprop" . "general|rightsinfo")
+      ("format" . "json")
+      ("formatversion" . "2")))]
+  [render
+   (define page-html (preprocess-html-wiki (jp "/parse/text" pagedata "")))
    (define page (html->xexp page-html))
-   (define head-data ((head-data-getter wikiname) page-data))
+   (define head-data ((head-data-getter wikiname) pagedata))
    (define body (generate-results-page
                  #:req req
                  #:source-url source-url
                  #:wikiname wikiname
-                 #:title title
+                 #:title (preprocess-html-wiki (jp "/parse/title" pagedata prefixed-category))
-                 #:members-data members-data
+                 #:members-data membersdata
                  #:page page
                  #:head-data head-data
                  #:siteinfo siteinfo))
@@ -116,7 +108,22 @@
     #:code 200
     #:headers (build-headers always-headers)
     (λ (out)
-      (write-html body out)))))
+      (write-html body out)))])
+
+(define-standard-handler (page-category-standard req)
+  #'category-endpoint)
+
+(define-jsonp-handler (page-category-jsonp req)
+  #'category-endpoint)
+
+(define-post-data-handler (page-category-with-data req)
+  #'category-endpoint
+  (define prefixed-category path))
+
+(define page-category (make-switch-handler #:standard page-category-standard
+                                           #:jsonp page-category-jsonp
+                                           #:post page-category-with-data))
+
 (module+ test
   (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor")
                                     (generate-results-page

src/page-file.rkt

@@ -18,6 +18,7 @@
          "fandom-request.rkt"
          "page-wiki.rkt"
          "../lib/syntax.rkt"
+         "../lib/tree-updater.rkt"
          "../lib/thread-utils.rkt"
          "../lib/url-utils.rkt"
          "../lib/xexpr-utils.rkt")

src/page-search.rkt

@@ -23,8 +23,7 @@
  page-search)
 
 (define search-providers
-  (hash "fandom" search-fandom
+  (hash "solr" search-solr))
-        "solr" search-solr))
 
 ;; this takes the info we gathered from fandom and makes the big fat x-expression page
 (define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f])
@@ -40,7 +39,7 @@
    results-content))
 
 ;; will be called when the web browser asks to load the page
-(define (page-search req)
+(define (page-search-solr req)
   ;; this just means, catch any errors and display them in the browser. it's a function somewhere else
   (response-handler
    ;; the URL will look like "/minecraft/wiki/Special:Search?q=Spawner"
@@ -84,3 +83,9 @@
     (λ (out)
       (write-html body out)))))
 
+
+
+(define (page-search req)
+  (if (equal? (config-get 'feature_offline::search) "fandom")
+      (page-search-fandom req)
+      (page-search-solr req)))

src/page-wiki-jsonp.rkt

@@ -1,72 +0,0 @@
-#lang racket/base
-(require racket/list
-         racket/string
-         web-server/http
-         net/url-structs
-         html-writing
-         "application-globals.rkt"
-         "data.rkt"
-         "config.rkt"
-         "../lib/url-utils.rkt"
-         "../lib/xexpr-utils.rkt"
-         "../lib/archive-file-mappings.rkt"
-         "static-data.rkt")
-
-(provide
- page-wiki-jsonp)
-
-(define (page-wiki-jsonp req)
-  (response-handler
-   (define wikiname (path/param-path (first (url-path (request-uri req)))))
-   (define segments (map path/param-path (cdr (url-path (request-uri req)))))
-   (define path (string-join (cdr segments) "/"))
-   (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
-
-   (define wiki-page-script-url
-     (format "https://~a.fandom.com/api.php?~a"
-             wikiname
-             (params->query `(("action" . "parse")
-                              ("page" . ,path)
-                              ("prop" . "text|headhtml|langlinks")
-                              ("formatversion" . "2")
-                              ("format" . "json")
-                              ("callback" . "proxy.wikipage")))))
-   (define siteinfo-script-url
-     (format "https://~a.fandom.com/api.php?~a"
-             wikiname
-             (params->query `(("action" . "query")
-                              ("meta" . "siteinfo")
-                              ("siprop" . "general|rightsinfo")
-                              ("format" . "json")
-                              ("formatversion" . "2")
-                              ("callback" . "proxy.siteinfo")))))
-
-   (define body
-     (generate-wiki-page
-      `(div
-        (noscript "You have to enable JavaScript to load wiki pages. Sorry!")
-        (div (@ (id "loading")))
-        (div (@ (id "progress-bar") (style "margin-bottom: 50vh"))
-             (progress))
-        (script #<<END
-var jsonpData = {}
-var proxy = new Proxy(jsonpData, {get(obj, prop) { return value => obj[prop] = value }})
-END
-                )
-        (script (@ (async) (src ,wiki-page-script-url) (onerror "proxy.wikipage({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})")))
-        (script (@ (async) (src ,siteinfo-script-url) (onerror "proxy.siteinfo({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})")))
-        (script (@ (type "module") (src ,(get-static-url "jsonp.js")))))
-      #:req req
-      #:source-url source-url
-      #:wikiname wikiname
-      #:title (url-segments->guess-title segments)
-      #:siteinfo siteinfo-default
-      #:path path
-      #:jsonp #t))
-   (when (config-true? 'debug)
-     (xexp->html body))
-   (response/output
-    #:code 200
-    #:headers always-headers
-    (λ (out)
-      (write-html body out)))))

src/page-wiki.rkt

@@ -1,12 +1,6 @@
 #lang racket/base
 (require racket/dict
-         racket/function
-         racket/list
-         racket/match
          racket/string
-         ; libs
-         (prefix-in easy: net/http-easy)
-         json
          ; html libs
          "../lib/html-parsing/main.rkt"
          html-writing
@@ -16,27 +10,121 @@
          web-server/dispatchers/dispatch
          ; my libs
          "application-globals.rkt"
-         "config.rkt"
+         "endpoints.rkt"
-         "data.rkt"
-         "fandom-request.rkt"
-         "../lib/archive-file-mappings.rkt"
-         "../lib/thread-utils.rkt"
          "../lib/tree-updater.rkt"
          "../lib/url-utils.rkt"
          "../lib/xexpr-utils.rkt")
+(require (for-syntax racket/base syntax/parse))
 
 (provide
- ; used by the web server
+ page-wiki)
- page-wiki
- page-wiki-with-data
- ; used by page-category, and similar pages that are partially wiki pages
- update-tree-wiki
- preprocess-html-wiki)
 
-(module+ test
+(define-endpoint
-  (require rackunit))
+  wiki-endpoint
+  [variables
+   (define wikiname (path/param-path (car (url-path (request-uri req)))))
+   (define segments (map path/param-path (cdr (url-path (request-uri req)))))
+   (define title (url-segments->guess-title segments))
+   (define path (string-join (cdr segments) "/"))
+   (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))]
+  [endpoints
+   (wikipage (("action" . "parse")
+              ("page" . ,path)
+              ("prop" . "text|headhtml|langlinks")
+              ("formatversion" . "2")
+              ("format" . "json")))
+   (siteinfo (("action" . "query")
+              ("meta" . "siteinfo")
+              ("siprop" . "general|rightsinfo")
+              ("format" . "json")
+              ("formatversion" . "2")))]
+  [render
+   (define page-html (preprocess-html-wiki (jp "/parse/text" wikipage "")))
+   (define page (html->xexp page-html))
+   (define head-data ((head-data-getter wikiname) wikipage))
+   (define body
+     (generate-wiki-page
+      (update-tree-wiki page wikiname)
+      #:req req
+      #:source-url source-url
+      #:wikiname wikiname
+      #:title (jp "/parse/title" wikipage "")
+      #:head-data head-data
+      #:siteinfo siteinfo))
+   (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
+   (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
+   (define redirect-msg-a (if redirect-msg
+                              ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
+                              #f))
+   (define html (xexp->html-bytes body))
+   (define headers
+     (build-headers
+      always-headers
+      ; redirect-query-parameter: only the string "no" is significant:
+      ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
+      (when (and redirect-msg-a
+                 (not (equal? redirect-query-parameter "no")))
+        (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
+               [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
+          (header #"Refresh" value)))))
+   (response/full
+    200
+    #"OK"
+    (current-seconds)
+    #"text/html; charset=utf-8"
+    headers
+    (list html))])
 
-(define (page-wiki req)
+(define-standard-handler (page-wiki-standard req)
+  #'wiki-endpoint
+  (when (equal? "missingtitle" (jp "/error/code" wikipage #f))
+    (next-dispatcher)))
+
+(define-jsonp-handler (page-wiki-jsonp req)
+  #'wiki-endpoint)
+
+(define-post-data-handler (page-wiki-with-data req)
+  #'wiki-endpoint)
+
+(define page-wiki (make-switch-handler #:standard page-wiki-standard
+                                       #:jsonp page-wiki-jsonp
+                                       #:post page-wiki-with-data))
+
+#;(define (page-wiki-with-data req)
+  (response-handler
+   (let/cc return
+     (define post-data/bytes (request-post-data/raw req))
+     (when (not post-data/bytes)
+       (k (response/jsexpr
+           #:code 400
+           #:headers always-headers
+           '#hasheq((error .
+                           #hasheq((code . "breezewiki")
+                                   (info . "POST requests only, please.")))))))
+
+     (define origin-header
+       (or (headers-assq* #"origin" (request-headers/raw req))
+           (headers-assq* #"referer" (request-headers/raw req))))
+     (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin))))
+       (k (response/jsexpr
+           #:code 400
+           #:headers always-headers
+           '#hasheq((error .
+                           #hasheq((code . "breezewiki")
+                                   (info . "Origin/Referer header failed validation - cross-origin requests are not allowed here")))))))
+
+     (define post-data/string (bytes->string/utf-8 post-data/bytes))
+     (define post-data (string->jsexpr post-data/string))
+     (define wikiname (jp "/wikiname" post-data))
+     (define path (jp "/path" post-data))
+     (take-json-rewrite-and-return-page
+      #:req req
+      #:wikiname wikiname
+      #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)
+      #:data (jp "/data" post-data)
+      #:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))))
+
+#;(define (page-wiki req)
   (define wikiname (path/param-path (first (url-path (request-uri req)))))
   (define segments (map path/param-path (cdr (url-path (request-uri req)))))
   (define user-cookies (user-cookies-getter req))
@@ -95,75 +183,3 @@
              (easy:response-status-code dest-res)
              (easy:response-body dest-res)))]))
 
-(define (page-wiki-with-data req)
-  (response-handler
-   (let/cc return
-     (define post-data/bytes (request-post-data/raw req))
-     (when (not post-data/bytes)
-       (return (response/jsexpr
-                #:code 400
-                #:headers always-headers
-                '#hasheq((error .
-                                #hasheq((code . "breezewiki")
-                                         (info . "POST requests only, please.")))))))
-
-     (define origin-header
-       (or (headers-assq* #"origin" (request-headers/raw req))
-           (headers-assq* #"referer" (request-headers/raw req))))
-     (when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin))))
-       (return (response/jsexpr
-                #:code 400
-                #:headers always-headers
-                '#hasheq((error .
-                                #hasheq((code . "breezewiki")
-                                        (info . "Origin/Referer header failed validation - cross-origin requests are not allowed here")))))))
-
-     (define post-data/string (bytes->string/utf-8 post-data/bytes))
-     (define post-data (string->jsexpr post-data/string))
-     (define wikiname (jp "/wikiname" post-data))
-     (define path (jp "/path" post-data))
-     (take-json-rewrite-and-return-page
-      #:req req
-      #:wikiname wikiname
-      #:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)
-      #:data (jp "/data" post-data)
-      #:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))))
-
-(define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo)
-  (define title (jp "/parse/title" data ""))
-  (define page-html (preprocess-html-wiki (jp "/parse/text" data "")))
-  (define page (html->xexp page-html))
-  (define head-data ((head-data-getter wikiname) data))
-  (response-handler
-   (define body
-     (generate-wiki-page
-      (update-tree-wiki page wikiname)
-      #:req req
-      #:source-url source-url
-      #:wikiname wikiname
-      #:title title
-      #:head-data head-data
-      #:siteinfo siteinfo))
-   (define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
-   (define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
-   (define redirect-msg-a (if redirect-msg
-                              ((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
-                              #f))
-   (define html (xexp->html-bytes body))
-   (define headers
-     (build-headers
-      always-headers
-      ; redirect-query-parameter: only the string "no" is significant:
-      ; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
-      (when (and redirect-msg-a
-                 (not (equal? redirect-query-parameter "no")))
-        (let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
-               [value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
-          (header #"Refresh" value)))))
-   (response/full
-    200
-    #"OK"
-    (current-seconds)
-    #"text/html; charset=utf-8"
-    headers
-    (list html))))

src/search-provider-fandom.rkt

@@ -1,59 +1,95 @@
 #lang racket/base
-(require racket/string
+(require racket/dict
+         racket/string
          (prefix-in easy: net/http-easy)
+         net/url
+         web-server/http
+         html-writing
          "application-globals.rkt"
          "config.rkt"
+         "endpoints.rkt"
          "fandom-request.rkt"
          "../lib/url-utils.rkt"
          "../lib/xexpr-utils.rkt")
 
 (provide
- search-fandom)
+ page-search-fandom)
 
-(module+ test
+(define-endpoint
-  (require rackunit
+  search-endpoint
-           "test-utils.rkt")
+  [variables
-  (define search-results-data
+   (define wikiname (path/param-path (car (url-path (request-uri req)))))
-    '(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
+   (define params (url-query (request-uri req)))
+   (define query (dict-ref params 'q #f))
+   (define title "Search")
+   (define path (format "Special:Search?~a" (params->query `(("query" . ,query)
+                                                             ("search" . "internal")))))
+   (define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))]
+  [endpoints
+   (search
+    (("action" . "query")
+     ("list" . "search")
+     ("srsearch" . ,query)
+     ("formatversion" . "2")
+     ("format" . "json")))
+   (siteinfo
+    (("action" . "query")
+     ("meta" . "siteinfo")
+     ("siprop" . "general|rightsinfo")
+     ("format" . "json")
+     ("formatversion" . "2")))]
+  [render
+   (define search-results (jp "/query/search" search))
+   (define body
+     (generate-wiki-page
+      #:req req
+      #:source-url source-url
+      #:wikiname wikiname
+      #:title query
+      #:siteinfo siteinfo
+      `(div (@ (class "mw-parser-output"))
+            ;; header before the search results showing how many we found
+            (p ,(format "~a results found for " (length search-results))
+               (strong ,query))
+            ;; *u*nordered *l*ist of matching search results
+            (ul ,@(for/list ([result search-results])
+                    (let* ([title (jp "/title" result)]
+                           [page-path (page-title->path title)]
+                           [timestamp (jp "/timestamp" result)]
+                           [wordcount (jp "/wordcount" result)]
+                           [size (jp "/size" result)])
+                      ;; and make this x-expression...
+                      `(li (@ (class "my-result"))
+                           (a (@ (class "my-result__link") (href ,(format "/~a/wiki/~a" wikiname page-path))) ; using unquote to insert the result page URL
+                              ,title) ; using unquote to insert the result page title
+                           (div (@ (class "my-result__info")) ; constructing the line under the search result
+                                "last edited "
+                                (time (@ (datetime ,timestamp)) ,(list-ref (string-split timestamp "T") 0))
+                                ,(format ", ~a words, ~a kb"
+                                         wordcount
+                                         (exact->inexact (/ (round (/ size 100)) 10)))))))))))
+   (when (config-true? 'debug)
+     (xexp->html body))
+   (response/output
+    #:code 200
+    #:headers (build-headers always-headers)
+    (λ (out)
+      (write-html body out)))])
 
-(define (search-fandom wikiname query params)
-  (define res
-    (fandom-get-api
-     wikiname
-     `(("action" . "query")
-       ("list" . "search")
-       ("srsearch" . ,query)
-       ("formatversion" . "2")
-       ("format" . "json"))))
-  (define json (easy:response-json res))
-  (define search-results (jp "/query/search" json))
-  (generate-results-content-fandom wikiname query search-results))
 
-;;; generate content for display in the wiki page layout
+(define-standard-handler (page-search-standard req)
-(define (generate-results-content-fandom wikiname query search-results)
+  #'search-endpoint)
-  `(div (@ (class "mw-parser-output"))
-        ;; header before the search results showing how many we found
-        (p ,(format "~a results found for " (length search-results))
-           (strong ,query))
-        ;; *u*nordered *l*ist of matching search results
-        (ul ,@(for/list ([result search-results])
-                (let* ([title (jp "/title" result)]
-                       [page-path (page-title->path title)]
-                       [timestamp (jp "/timestamp" result)]
-                       [wordcount (jp "/wordcount" result)]
-                       [size (jp "/size" result)])
-                  ;; and make this x-expression...
-                  `(li (@ (class "my-result"))
-                       (a (@ (class "my-result__link") (href ,(format "/~a/wiki/~a" wikiname page-path))) ; using unquote to insert the result page URL
-                          ,title) ; using unquote to insert the result page title
-                       (div (@ (class "my-result__info")) ; constructing the line under the search result
-                            "last edited "
-                            (time (@ (datetime ,timestamp)) ,(list-ref (string-split timestamp "T") 0))
-                            ,(format ", ~a words, ~a kb"
-                                     wordcount
-                                     (exact->inexact (/ (round (/ size 100)) 10))))))))))
 
-(module+ test
+(define-jsonp-handler (page-search-jsonp req)
-  (parameterize ([(config-parameter 'feature_offline::only) "false"])
+  #'search-endpoint)
-    (check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule")
+
-                                      (generate-results-content-fandom "test" "Gacha" search-results-data))))))
+(define-post-data-handler (page-search-with-data req)
+  #'search-endpoint
+  (define params (url-query (request-uri req)))
+  (define query (dict-ref params 'q #f))
+  (define title "Search"))
+
+(define page-search-fandom
+  (make-switch-handler #:standard page-search-standard
+                       #:jsonp page-search-jsonp
+                       #:post page-search-with-data))

static/jsonp.js

@@ -37,23 +37,29 @@ window.proxy = new Proxy(jsonpData, {
 // *** Data upload and download
 
 async function cont() {
-	if (jsonpData.wikipage?.error) return error(jsonpData.wikipage)
+	// Check for errors
-	if (jsonpData.siteinfo?.error) return error(jsonpData.siteinfo)
+	for (const v of Object.values(jsonpData)) {
-	if (!(jsonpData.wikipage && jsonpData.siteinfo)) return
+		if (v.error) return error(v)
+	}
+
+	// Check for completion
+	const dependencies = [...document.querySelectorAll("[data-jsonp-var]")].map(e => e.getAttribute("data-jsonp-var"))
+	for (const d of dependencies) {
+		if (!(d in jsonpData)) return
+	}
 
 	const xhr = new XMLHttpRequest();
 
 	const uploadFraction = 0.7
 
 	const pkg = {
-		url: "/api/render/wiki",
+		url: location.href,
 		init: {
 			method: "POST",
 			body: JSON.stringify({
-				data: jsonpData.wikipage,
-				siteinfo: jsonpData.siteinfo,
 				wikiname: BWData.wikiname,
-				path: BWData.path
+				path: BWData.path,
+				...jsonpData
 			})
 		}
 	}
@@ -127,7 +133,7 @@ function error(data) {
 	if (typeof data === "string") {
 		render(html`<p><strong>BreezeWiki ran into an error on this page.</strong></p><p>Try reloading the page.</p><p>If this keeps happening, you could <a href="mailto:~cadence/breezewiki-discuss@lists.sr.ht">send a public bug report</a>. Please include the following information:</p><pre>URL: ${window.location.href}${"\n"}${data}</pre>`, eContent)
 	} else if (data.error.code === "missingtitle") {
-		render(html`<p><strong>This page doesn't exist on Fandom.</strong></p><p><small><a href="/${BWData.wikiname}/wiki/Main_Page">Return to the homepage?</a></small></p>`, eContent)
+		render(html`<p><strong>This page doesn't exist on Fandom.</strong></p><p><small><a href="/${BWData.wikiname}/wiki/Main_Page">Return to the wiki's main page</a></small></p>`, eContent)
 	} else {
 		render(html`<p>BreezeWiki wasn't able to load this page.</p><p><strong>${data.error.code}: ${data.error.info}</strong></p>`, eContent)
 	}