mirror of
https://gitdab.com/cadence/breezewiki.git
synced 2026-03-04 13:30:04 -05:00
Add JSONP support on other endpoints
Refactored the code to define and process endpoints rather than doing it imperatively, which avoided a lot of duplicated code.
This commit is contained in:
Cadence Ember
@@ -27,9 +27,7 @@
|
||||
(require-reloadable "src/page-static-archive.rkt" page-static-archive)
|
||||
(require-reloadable "src/page-subdomain.rkt" subdomain-dispatcher)
|
||||
(require-reloadable "src/page-wiki.rkt" page-wiki)
|
||||
(require-reloadable "src/page-wiki.rkt" page-wiki-with-data)
|
||||
(require-reloadable "src/page-wiki-offline.rkt" page-wiki-offline)
|
||||
(require-reloadable "src/page-wiki-jsonp.rkt" page-wiki-jsonp)
|
||||
(require-reloadable "src/page-file.rkt" page-file)
|
||||
|
||||
(reload!)
|
||||
@@ -59,9 +57,7 @@
|
||||
page-set-user-settings
|
||||
page-static-archive
|
||||
page-wiki
|
||||
page-wiki-with-data
|
||||
page-wiki-offline
|
||||
page-wiki-jsonp
|
||||
page-file
|
||||
redirect-wiki-home
|
||||
static-dispatcher
|
||||
|
||||
5
dist.rkt
5
dist.rkt
@@ -16,9 +16,8 @@
|
||||
(require (only-in "src/page-static.rkt" static-dispatcher))
|
||||
(require (only-in "src/page-static-archive.rkt" page-static-archive))
|
||||
(require (only-in "src/page-subdomain.rkt" subdomain-dispatcher))
|
||||
(require (only-in "src/page-wiki.rkt" page-wiki page-wiki-with-data))
|
||||
(require (only-in "src/page-wiki.rkt" page-wiki))
|
||||
(require (only-in "src/page-wiki-offline.rkt" page-wiki-offline))
|
||||
(require (only-in "src/page-wiki-jsonp.rkt" page-wiki-jsonp))
|
||||
(require (only-in "src/page-file.rkt" page-file))
|
||||
|
||||
(serve/launch/wait
|
||||
@@ -43,8 +42,6 @@
|
||||
page-static-archive
|
||||
page-wiki
|
||||
page-wiki-offline
|
||||
page-wiki-with-data
|
||||
page-wiki-jsonp
|
||||
page-file
|
||||
redirect-wiki-home
|
||||
static-dispatcher
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
update-tree-wiki)
|
||||
|
||||
(define (preprocess-html-wiki html)
|
||||
(regexp-replace* #rx"(<(?:td|figcaption)[^>]*?>\n?)(?:<li>|[ \t]*?<p class=\"caption\">(.*?)</p>)"
|
||||
(regexp-replace* #rx"(<(?:td|figcaption)[^>]*?>\n?)(?:[ \t]*<a href=\"[^\"]*\" class=\"info-icon\"><svg><use xlink:href=\"#wds-icons-info-small\"></use></svg></a>)?(?:<li>|[ \t]*?<p class=\"caption\">(.*?)</p>)"
|
||||
html (λ (whole first-tag [contents #f])
|
||||
(if (eq? (string-ref whole 1) #\f) ;; figcaption
|
||||
(string-append first-tag "<span class=\"caption\">" contents "</span>")
|
||||
|
||||
@@ -55,6 +55,7 @@
|
||||
(define tree
|
||||
(sequencer:make
|
||||
subdomain-dispatcher
|
||||
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
|
||||
(pathprocedure:make "/" (page ds page-home))
|
||||
(filter:make #rx"^/static/" (hash-ref ds 'static-dispatcher))
|
||||
(filter:make (pregexp "^/captcha/img/[0-9]+/[0-9]+$") (lift:make (page ds page-captcha-image)))
|
||||
@@ -65,16 +66,11 @@
|
||||
(pathprocedure:make "/proxy" (page ds page-proxy))
|
||||
(pathprocedure:make "/search" (page ds page-global-search))
|
||||
(pathprocedure:make "/set-user-settings" (page ds page-set-user-settings))
|
||||
(pathprocedure:make "/buddyfight/wiki/It_Doesn't_Work!!" (page ds page-it-works))
|
||||
(pathprocedure:make "/api/render/wiki" (page ds page-wiki-with-data))
|
||||
(filter:make (pregexp (format "^/~a/wiki/Category:.+$" px-wikiname)) (lift:make (page ds page-category)))
|
||||
(filter:make (pregexp (format "^/~a/wiki/File:.+$" px-wikiname)) (lift:make (page ds page-file)))
|
||||
(if (config-true? 'feature_offline::enabled)
|
||||
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-offline)))
|
||||
(λ (_conn _req) (next-dispatcher)))
|
||||
(if (config-true? 'feature_jsonp::enabled)
|
||||
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki-jsonp)))
|
||||
(λ (_conn _req) (next-dispatcher)))
|
||||
(filter:make (pregexp (format "^/~a/wiki/.+$" px-wikiname)) (lift:make (page ds page-wiki)))
|
||||
(filter:make (pregexp (format "^/~a/search$" px-wikiname)) (lift:make (page ds page-search)))
|
||||
(filter:make (pregexp (format "^/~a(/(wiki(/)?)?)?$" px-wikiname)) (lift:make (page ds redirect-wiki-home)))
|
||||
|
||||
248
src/endpoints.rkt
Normal file
248
src/endpoints.rkt
Normal file
@@ -0,0 +1,248 @@
|
||||
#lang racket/base
|
||||
(require racket/format
|
||||
racket/match
|
||||
racket/string
|
||||
; libs
|
||||
(prefix-in easy: net/http-easy)
|
||||
json
|
||||
; html libs
|
||||
html-writing
|
||||
; web server libs
|
||||
web-server/http
|
||||
web-server/dispatchers/dispatch
|
||||
; my libs
|
||||
"application-globals.rkt"
|
||||
"config.rkt"
|
||||
"data.rkt"
|
||||
"fandom-request.rkt"
|
||||
"static-data.rkt"
|
||||
"../lib/archive-file-mappings.rkt"
|
||||
"../lib/thread-utils.rkt"
|
||||
"../lib/url-utils.rkt"
|
||||
"../lib/xexpr-utils.rkt")
|
||||
(require (for-syntax racket/base syntax/parse))
|
||||
|
||||
(provide
|
||||
define-endpoint
|
||||
define-standard-handler
|
||||
define-post-data-handler
|
||||
define-jsonp-handler
|
||||
make-switch-handler
|
||||
(all-from-out racket/format
|
||||
racket/match
|
||||
json
|
||||
net/http-easy
|
||||
web-server/dispatchers/dispatch
|
||||
"../lib/thread-utils.rkt"
|
||||
"../lib/archive-file-mappings.rkt"
|
||||
"config.rkt"
|
||||
"data.rkt"
|
||||
"fandom-request.rkt"
|
||||
"static-data.rkt"))
|
||||
|
||||
|
||||
(define-for-syntax (fix-scopes here orig stx)
|
||||
(define remover (make-syntax-delta-introducer here #f))
|
||||
(define introducer (make-syntax-delta-introducer orig #f))
|
||||
(introducer (remover stx 'remove)))
|
||||
|
||||
|
||||
(define-syntax (define-endpoint stx)
|
||||
(syntax-parse stx
|
||||
[(_ name
|
||||
(~and (~seq all ...)
|
||||
(~seq
|
||||
((~datum variables) variable ...)
|
||||
((~datum endpoints) endpoint ...)
|
||||
((~datum render) render ...))))
|
||||
#'(define-syntax name #'(all ...))]))
|
||||
|
||||
|
||||
(define ((make-switch-handler #:standard standard #:jsonp jsonp #:post post) req)
|
||||
(cond
|
||||
[(equal? (request-method req) #"POST")
|
||||
(post req)]
|
||||
[(config-true? 'feature_jsonp::enabled)
|
||||
(jsonp req)]
|
||||
[else
|
||||
(standard req)]))
|
||||
|
||||
|
||||
(define-syntax (define-standard-handler stx)
|
||||
(syntax-parse stx
|
||||
[(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...)
|
||||
#:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref)))
|
||||
(syntax-parse #'endpoint-syntax
|
||||
#:context 'endpoint-data
|
||||
[(((~datum variables) variable ...)
|
||||
((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...)
|
||||
((~datum render) render ...))
|
||||
#:with endpoint-getters (for/list ([id (syntax-e #'(endpoint-id ...))]
|
||||
[params (syntax-e #'((endpoint-param ...) ...))])
|
||||
(with-syntax ([params (append (syntax-e params) #;'(("callback" . "proxy.wikipage")))])
|
||||
(if (eq? (syntax-e id) 'siteinfo)
|
||||
#'(λ ()
|
||||
(siteinfo-fetch wikiname))
|
||||
#'(λ ()
|
||||
(fandom-get-api
|
||||
wikiname
|
||||
`params
|
||||
#:headers `#hasheq((cookie . ,(format "theme=~a" (user-cookies^-theme user-cookies)))))))))
|
||||
#:with endpoint-intermediates->values (for/list ([id (syntax-e #'(endpoint-id ...))])
|
||||
(with-syntax ([id id])
|
||||
(if (eq? (syntax-e #'id) 'siteinfo)
|
||||
#'id
|
||||
#'(easy:response-json id))))
|
||||
|
||||
(fix-scopes
|
||||
#'here stx
|
||||
#'(define name-args
|
||||
(response-handler
|
||||
(let/cc k
|
||||
variable ...
|
||||
(define user-cookies (user-cookies-getter req))
|
||||
(define-values (endpoint-id ...)
|
||||
(let-values ([(endpoint-id ...) (thread-values (~@ . endpoint-getters))])
|
||||
(for ([response (list endpoint-id ...)]
|
||||
#:when (easy:response? response))
|
||||
(match (easy:response-status-code response)
|
||||
[404
|
||||
(next-dispatcher)]
|
||||
[(or 403 406)
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
`(div
|
||||
(p "Sorry! Fandom isn't allowing BreezeWiki to show pages right now.")
|
||||
(p "We'll automatically try again in 30 seconds, so please stay on this page and be patient.")
|
||||
(p (small "In a hurry? " (a (@ (href ,source-url)) "Click here to read the page on Fandom."))))
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title title
|
||||
#:siteinfo siteinfo))
|
||||
(k (response/output
|
||||
#:code 503
|
||||
#:headers (build-headers
|
||||
always-headers
|
||||
(header #"Retry-After" #"30")
|
||||
(header #"Cache-Control" #"max-age=30, public")
|
||||
(header #"Refresh" #"35"))
|
||||
(λ (out)
|
||||
(write-html body out))))]
|
||||
[(not 200)
|
||||
(k (error 'page-wiki "Tried to load page ~a/~a~nSadly, the page didn't load because Fandom returned status code ~a with response:~n~a"
|
||||
wikiname
|
||||
path
|
||||
(easy:response-status-code response)
|
||||
(easy:response-body response)))]
|
||||
[_ (void)]))
|
||||
(values (~@ . endpoint-intermediates->values))))
|
||||
fn-body ...
|
||||
render ...))))])]))
|
||||
|
||||
|
||||
(define-syntax (define-post-data-handler stx)
|
||||
(syntax-parse stx
|
||||
[(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...)
|
||||
#:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref)))
|
||||
(syntax-parse #'endpoint-syntax
|
||||
#:context 'endpoint-data
|
||||
[(((~datum variables) variable ...)
|
||||
((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...)
|
||||
((~datum render) render ...))
|
||||
#:with endpoint-getters (for/list ([id (syntax-e #'(endpoint-id ...))]
|
||||
[params (syntax-e #'((endpoint-param ...) ...))])
|
||||
(with-syntax ([id id])
|
||||
(if (eq? (syntax-e #'id) 'siteinfo)
|
||||
#'(data->siteinfo (hash-ref post-data 'id))
|
||||
#'(hash-ref post-data 'id))))
|
||||
|
||||
(fix-scopes
|
||||
#'here stx
|
||||
#'(define name-args
|
||||
(response-handler
|
||||
(let/cc k
|
||||
(define (k-validation-error message)
|
||||
(k (response/jsexpr
|
||||
#:code 400
|
||||
#:headers always-headers
|
||||
`#hasheq((error .
|
||||
#hasheq((code . "breezewiki")
|
||||
(info . ,message)))))))
|
||||
(define post-data/bytes (request-post-data/raw req))
|
||||
(when (not post-data/bytes)
|
||||
(k-validation-error "POST requests only, please."))
|
||||
|
||||
(define origin-header
|
||||
(or (headers-assq* #"origin" (request-headers/raw req))
|
||||
(headers-assq* #"referer" (request-headers/raw req))))
|
||||
(when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin))))
|
||||
(k-validation-error "Origin/Referer header failed validation - cross-origin requests are not allowed here"))
|
||||
|
||||
(define post-data/string (bytes->string/utf-8 post-data/bytes))
|
||||
(define post-data (string->jsexpr post-data/string))
|
||||
(define-values (endpoint-id ...) (values (~@ . endpoint-getters)))
|
||||
(define wikiname (hash-ref post-data 'wikiname))
|
||||
(define path (hash-ref post-data 'path))
|
||||
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
|
||||
fn-body ...
|
||||
render ...))))])]))
|
||||
|
||||
|
||||
(define-syntax (define-jsonp-handler stx)
|
||||
(syntax-parse stx
|
||||
[(_ (~and name-args (name:id arg:id ...)) endpoint-ref:expr fn-body ...)
|
||||
#:with endpoint-syntax (syntax-local-value (cadr (syntax->list #'endpoint-ref)))
|
||||
(syntax-parse #'endpoint-syntax
|
||||
#:context 'endpoint-data
|
||||
[(((~datum variables) variable ...)
|
||||
((~datum endpoints) (endpoint-id:id (endpoint-param:expr ...)) ...)
|
||||
((~datum render) render ...))
|
||||
#:with endpoint-scripts (for/list ([id (syntax-e #'(endpoint-id ...))]
|
||||
[params (syntax-e #'((endpoint-param ...) ...))])
|
||||
(with-syntax ([id id]
|
||||
[params (append (syntax-e params)
|
||||
`(("callback" . ,(format "proxy.~a" (syntax-e id)))))])
|
||||
#'(script (@ (async)
|
||||
(src ,(format "https://~a.fandom.com/api.php?~a"
|
||||
wikiname
|
||||
(params->query `params)))
|
||||
(data-jsonp-var ,(format "~a" 'id))
|
||||
(onerror ,(format "proxy.~a({error: {code: 'disconnected', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})" 'id))))))
|
||||
|
||||
(fix-scopes
|
||||
#'here stx
|
||||
#'(define name-args
|
||||
(response-handler
|
||||
(let/cc k
|
||||
variable ...
|
||||
fn-body ...
|
||||
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
`(div
|
||||
(noscript "You have to enable JavaScript to load wiki pages. Sorry!")
|
||||
(div (@ (id "loading")))
|
||||
(div (@ (id "progress-bar") (style "margin-bottom: 50vh"))
|
||||
(progress))
|
||||
(script #<<END
|
||||
var jsonpData = {}
|
||||
var proxy = new Proxy(jsonpData, {get(obj, prop) { return value => obj[prop] = value }})
|
||||
END
|
||||
)
|
||||
endpoint-scripts
|
||||
(script (@ (type "module") (src ,(get-static-url "jsonp.js")))))
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title title
|
||||
#:siteinfo siteinfo-default
|
||||
#:path path
|
||||
#:jsonp #t))
|
||||
(when (config-true? 'debug)
|
||||
(xexp->html body))
|
||||
(response/output
|
||||
#:code 200
|
||||
#:headers always-headers
|
||||
(λ (out)
|
||||
(write-html body out)))))))])]))
|
||||
@@ -1,8 +1,5 @@
|
||||
#lang racket/base
|
||||
(require racket/dict
|
||||
racket/list
|
||||
racket/match
|
||||
racket/string
|
||||
(require racket/string
|
||||
(prefix-in easy: net/http-easy)
|
||||
; html libs
|
||||
html-parsing
|
||||
@@ -11,14 +8,9 @@
|
||||
net/url
|
||||
web-server/http
|
||||
(only-in web-server/dispatchers/dispatch next-dispatcher)
|
||||
#;(only-in web-server/http/redirect redirect-to)
|
||||
"application-globals.rkt"
|
||||
"config.rkt"
|
||||
"data.rkt"
|
||||
"fandom-request.rkt"
|
||||
"page-wiki.rkt"
|
||||
"../lib/syntax.rkt"
|
||||
"../lib/thread-utils.rkt"
|
||||
"endpoints.rkt"
|
||||
"../lib/tree-updater.rkt"
|
||||
"../lib/url-utils.rkt"
|
||||
"../lib/xexpr-utils.rkt")
|
||||
|
||||
@@ -63,47 +55,47 @@
|
||||
,title)))
|
||||
members))))))
|
||||
|
||||
(define (page-category req)
|
||||
(response-handler
|
||||
(define wikiname (path/param-path (first (url-path (request-uri req)))))
|
||||
(define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
|
||||
(define origin (format "https://~a.fandom.com" wikiname))
|
||||
(define source-url (format "~a/wiki/~a" origin prefixed-category))
|
||||
|
||||
(define-values (members-data page-data siteinfo)
|
||||
(thread-values
|
||||
(λ ()
|
||||
(easy:response-json
|
||||
(fandom-get-api
|
||||
wikiname
|
||||
`(("action" . "query")
|
||||
(define-endpoint
|
||||
category-endpoint
|
||||
[variables
|
||||
(define wikiname (path/param-path (car (url-path (request-uri req)))))
|
||||
(define prefixed-category (string-join (map path/param-path (cddr (url-path (request-uri req)))) "/"))
|
||||
(define segments (map path/param-path (cdr (url-path (request-uri req)))))
|
||||
(define title (url-segments->guess-title segments))
|
||||
(define path (string-join (cdr segments) "/"))
|
||||
(define origin (format "https://~a.fandom.com" wikiname))
|
||||
(define source-url (format "~a/wiki/~a" origin prefixed-category))]
|
||||
[endpoints
|
||||
(membersdata
|
||||
(("action" . "query")
|
||||
("list" . "categorymembers")
|
||||
("cmtitle" . ,prefixed-category)
|
||||
("cmlimit" . "max")
|
||||
("formatversion" . "2")
|
||||
("format" . "json")))))
|
||||
(λ ()
|
||||
(easy:response-json
|
||||
(fandom-get-api
|
||||
wikiname
|
||||
`(("action" . "parse")
|
||||
("format" . "json")))
|
||||
(pagedata
|
||||
(("action" . "parse")
|
||||
("page" . ,prefixed-category)
|
||||
("prop" . "text|headhtml|langlinks")
|
||||
("formatversion" . "2")
|
||||
("format" . "json")))))
|
||||
(λ ()
|
||||
(siteinfo-fetch wikiname))))
|
||||
|
||||
(define title (preprocess-html-wiki (jp "/parse/title" page-data prefixed-category)))
|
||||
(define page-html (preprocess-html-wiki (jp "/parse/text" page-data "")))
|
||||
("format" . "json")))
|
||||
(siteinfo
|
||||
(("action" . "query")
|
||||
("meta" . "siteinfo")
|
||||
("siprop" . "general|rightsinfo")
|
||||
("format" . "json")
|
||||
("formatversion" . "2")))]
|
||||
[render
|
||||
(define page-html (preprocess-html-wiki (jp "/parse/text" pagedata "")))
|
||||
(define page (html->xexp page-html))
|
||||
(define head-data ((head-data-getter wikiname) page-data))
|
||||
(define head-data ((head-data-getter wikiname) pagedata))
|
||||
(define body (generate-results-page
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title title
|
||||
#:members-data members-data
|
||||
#:title (preprocess-html-wiki (jp "/parse/title" pagedata prefixed-category))
|
||||
#:members-data membersdata
|
||||
#:page page
|
||||
#:head-data head-data
|
||||
#:siteinfo siteinfo))
|
||||
@@ -116,7 +108,22 @@
|
||||
#:code 200
|
||||
#:headers (build-headers always-headers)
|
||||
(λ (out)
|
||||
(write-html body out)))))
|
||||
(write-html body out)))])
|
||||
|
||||
(define-standard-handler (page-category-standard req)
|
||||
#'category-endpoint)
|
||||
|
||||
(define-jsonp-handler (page-category-jsonp req)
|
||||
#'category-endpoint)
|
||||
|
||||
(define-post-data-handler (page-category-with-data req)
|
||||
#'category-endpoint
|
||||
(define prefixed-category path))
|
||||
|
||||
(define page-category (make-switch-handler #:standard page-category-standard
|
||||
#:jsonp page-category-jsonp
|
||||
#:post page-category-with-data))
|
||||
|
||||
(module+ test
|
||||
(check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Ankle_Monitor")
|
||||
(generate-results-page
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
"fandom-request.rkt"
|
||||
"page-wiki.rkt"
|
||||
"../lib/syntax.rkt"
|
||||
"../lib/tree-updater.rkt"
|
||||
"../lib/thread-utils.rkt"
|
||||
"../lib/url-utils.rkt"
|
||||
"../lib/xexpr-utils.rkt")
|
||||
|
||||
@@ -23,8 +23,7 @@
|
||||
page-search)
|
||||
|
||||
(define search-providers
|
||||
(hash "fandom" search-fandom
|
||||
"solr" search-solr))
|
||||
(hash "solr" search-solr))
|
||||
|
||||
;; this takes the info we gathered from fandom and makes the big fat x-expression page
|
||||
(define (generate-results-page req source-url wikiname query results-content #:siteinfo [siteinfo #f])
|
||||
@@ -40,7 +39,7 @@
|
||||
results-content))
|
||||
|
||||
;; will be called when the web browser asks to load the page
|
||||
(define (page-search req)
|
||||
(define (page-search-solr req)
|
||||
;; this just means, catch any errors and display them in the browser. it's a function somewhere else
|
||||
(response-handler
|
||||
;; the URL will look like "/minecraft/wiki/Special:Search?q=Spawner"
|
||||
@@ -84,3 +83,9 @@
|
||||
(λ (out)
|
||||
(write-html body out)))))
|
||||
|
||||
|
||||
|
||||
(define (page-search req)
|
||||
(if (equal? (config-get 'feature_offline::search) "fandom")
|
||||
(page-search-fandom req)
|
||||
(page-search-solr req)))
|
||||
@@ -1,72 +0,0 @@
|
||||
#lang racket/base
|
||||
(require racket/list
|
||||
racket/string
|
||||
web-server/http
|
||||
net/url-structs
|
||||
html-writing
|
||||
"application-globals.rkt"
|
||||
"data.rkt"
|
||||
"config.rkt"
|
||||
"../lib/url-utils.rkt"
|
||||
"../lib/xexpr-utils.rkt"
|
||||
"../lib/archive-file-mappings.rkt"
|
||||
"static-data.rkt")
|
||||
|
||||
(provide
|
||||
page-wiki-jsonp)
|
||||
|
||||
(define (page-wiki-jsonp req)
|
||||
(response-handler
|
||||
(define wikiname (path/param-path (first (url-path (request-uri req)))))
|
||||
(define segments (map path/param-path (cdr (url-path (request-uri req)))))
|
||||
(define path (string-join (cdr segments) "/"))
|
||||
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))
|
||||
|
||||
(define wiki-page-script-url
|
||||
(format "https://~a.fandom.com/api.php?~a"
|
||||
wikiname
|
||||
(params->query `(("action" . "parse")
|
||||
("page" . ,path)
|
||||
("prop" . "text|headhtml|langlinks")
|
||||
("formatversion" . "2")
|
||||
("format" . "json")
|
||||
("callback" . "proxy.wikipage")))))
|
||||
(define siteinfo-script-url
|
||||
(format "https://~a.fandom.com/api.php?~a"
|
||||
wikiname
|
||||
(params->query `(("action" . "query")
|
||||
("meta" . "siteinfo")
|
||||
("siprop" . "general|rightsinfo")
|
||||
("format" . "json")
|
||||
("formatversion" . "2")
|
||||
("callback" . "proxy.siteinfo")))))
|
||||
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
`(div
|
||||
(noscript "You have to enable JavaScript to load wiki pages. Sorry!")
|
||||
(div (@ (id "loading")))
|
||||
(div (@ (id "progress-bar") (style "margin-bottom: 50vh"))
|
||||
(progress))
|
||||
(script #<<END
|
||||
var jsonpData = {}
|
||||
var proxy = new Proxy(jsonpData, {get(obj, prop) { return value => obj[prop] = value }})
|
||||
END
|
||||
)
|
||||
(script (@ (async) (src ,wiki-page-script-url) (onerror "proxy.wikipage({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})")))
|
||||
(script (@ (async) (src ,siteinfo-script-url) (onerror "proxy.siteinfo({error: {code: 'script blocked', info: 'Fandom connection failed or was blocked by your browser. Check any browser extensions that may block third-party scripts, then reload the page.'}})")))
|
||||
(script (@ (type "module") (src ,(get-static-url "jsonp.js")))))
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title (url-segments->guess-title segments)
|
||||
#:siteinfo siteinfo-default
|
||||
#:path path
|
||||
#:jsonp #t))
|
||||
(when (config-true? 'debug)
|
||||
(xexp->html body))
|
||||
(response/output
|
||||
#:code 200
|
||||
#:headers always-headers
|
||||
(λ (out)
|
||||
(write-html body out)))))
|
||||
@@ -1,12 +1,6 @@
|
||||
#lang racket/base
|
||||
(require racket/dict
|
||||
racket/function
|
||||
racket/list
|
||||
racket/match
|
||||
racket/string
|
||||
; libs
|
||||
(prefix-in easy: net/http-easy)
|
||||
json
|
||||
; html libs
|
||||
"../lib/html-parsing/main.rkt"
|
||||
html-writing
|
||||
@@ -16,27 +10,121 @@
|
||||
web-server/dispatchers/dispatch
|
||||
; my libs
|
||||
"application-globals.rkt"
|
||||
"config.rkt"
|
||||
"data.rkt"
|
||||
"fandom-request.rkt"
|
||||
"../lib/archive-file-mappings.rkt"
|
||||
"../lib/thread-utils.rkt"
|
||||
"endpoints.rkt"
|
||||
"../lib/tree-updater.rkt"
|
||||
"../lib/url-utils.rkt"
|
||||
"../lib/xexpr-utils.rkt")
|
||||
(require (for-syntax racket/base syntax/parse))
|
||||
|
||||
(provide
|
||||
; used by the web server
|
||||
page-wiki
|
||||
page-wiki-with-data
|
||||
; used by page-category, and similar pages that are partially wiki pages
|
||||
update-tree-wiki
|
||||
preprocess-html-wiki)
|
||||
page-wiki)
|
||||
|
||||
(module+ test
|
||||
(require rackunit))
|
||||
(define-endpoint
|
||||
wiki-endpoint
|
||||
[variables
|
||||
(define wikiname (path/param-path (car (url-path (request-uri req)))))
|
||||
(define segments (map path/param-path (cdr (url-path (request-uri req)))))
|
||||
(define title (url-segments->guess-title segments))
|
||||
(define path (string-join (cdr segments) "/"))
|
||||
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))]
|
||||
[endpoints
|
||||
(wikipage (("action" . "parse")
|
||||
("page" . ,path)
|
||||
("prop" . "text|headhtml|langlinks")
|
||||
("formatversion" . "2")
|
||||
("format" . "json")))
|
||||
(siteinfo (("action" . "query")
|
||||
("meta" . "siteinfo")
|
||||
("siprop" . "general|rightsinfo")
|
||||
("format" . "json")
|
||||
("formatversion" . "2")))]
|
||||
[render
|
||||
(define page-html (preprocess-html-wiki (jp "/parse/text" wikipage "")))
|
||||
(define page (html->xexp page-html))
|
||||
(define head-data ((head-data-getter wikiname) wikipage))
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
(update-tree-wiki page wikiname)
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title (jp "/parse/title" wikipage "")
|
||||
#:head-data head-data
|
||||
#:siteinfo siteinfo))
|
||||
(define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
|
||||
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
|
||||
(define redirect-msg-a (if redirect-msg
|
||||
((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
|
||||
#f))
|
||||
(define html (xexp->html-bytes body))
|
||||
(define headers
|
||||
(build-headers
|
||||
always-headers
|
||||
; redirect-query-parameter: only the string "no" is significant:
|
||||
; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
|
||||
(when (and redirect-msg-a
|
||||
(not (equal? redirect-query-parameter "no")))
|
||||
(let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
|
||||
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
|
||||
(header #"Refresh" value)))))
|
||||
(response/full
|
||||
200
|
||||
#"OK"
|
||||
(current-seconds)
|
||||
#"text/html; charset=utf-8"
|
||||
headers
|
||||
(list html))])
|
||||
|
||||
(define (page-wiki req)
|
||||
(define-standard-handler (page-wiki-standard req)
|
||||
#'wiki-endpoint
|
||||
(when (equal? "missingtitle" (jp "/error/code" wikipage #f))
|
||||
(next-dispatcher)))
|
||||
|
||||
(define-jsonp-handler (page-wiki-jsonp req)
|
||||
#'wiki-endpoint)
|
||||
|
||||
(define-post-data-handler (page-wiki-with-data req)
|
||||
#'wiki-endpoint)
|
||||
|
||||
(define page-wiki (make-switch-handler #:standard page-wiki-standard
|
||||
#:jsonp page-wiki-jsonp
|
||||
#:post page-wiki-with-data))
|
||||
|
||||
#;(define (page-wiki-with-data req)
|
||||
(response-handler
|
||||
(let/cc return
|
||||
(define post-data/bytes (request-post-data/raw req))
|
||||
(when (not post-data/bytes)
|
||||
(k (response/jsexpr
|
||||
#:code 400
|
||||
#:headers always-headers
|
||||
'#hasheq((error .
|
||||
#hasheq((code . "breezewiki")
|
||||
(info . "POST requests only, please.")))))))
|
||||
|
||||
(define origin-header
|
||||
(or (headers-assq* #"origin" (request-headers/raw req))
|
||||
(headers-assq* #"referer" (request-headers/raw req))))
|
||||
(when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin))))
|
||||
(k (response/jsexpr
|
||||
#:code 400
|
||||
#:headers always-headers
|
||||
'#hasheq((error .
|
||||
#hasheq((code . "breezewiki")
|
||||
(info . "Origin/Referer header failed validation - cross-origin requests are not allowed here")))))))
|
||||
|
||||
(define post-data/string (bytes->string/utf-8 post-data/bytes))
|
||||
(define post-data (string->jsexpr post-data/string))
|
||||
(define wikiname (jp "/wikiname" post-data))
|
||||
(define path (jp "/path" post-data))
|
||||
(take-json-rewrite-and-return-page
|
||||
#:req req
|
||||
#:wikiname wikiname
|
||||
#:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)
|
||||
#:data (jp "/data" post-data)
|
||||
#:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))))
|
||||
|
||||
#;(define (page-wiki req)
|
||||
(define wikiname (path/param-path (first (url-path (request-uri req)))))
|
||||
(define segments (map path/param-path (cdr (url-path (request-uri req)))))
|
||||
(define user-cookies (user-cookies-getter req))
|
||||
@@ -95,75 +183,3 @@
|
||||
(easy:response-status-code dest-res)
|
||||
(easy:response-body dest-res)))]))
|
||||
|
||||
(define (page-wiki-with-data req)
|
||||
(response-handler
|
||||
(let/cc return
|
||||
(define post-data/bytes (request-post-data/raw req))
|
||||
(when (not post-data/bytes)
|
||||
(return (response/jsexpr
|
||||
#:code 400
|
||||
#:headers always-headers
|
||||
'#hasheq((error .
|
||||
#hasheq((code . "breezewiki")
|
||||
(info . "POST requests only, please.")))))))
|
||||
|
||||
(define origin-header
|
||||
(or (headers-assq* #"origin" (request-headers/raw req))
|
||||
(headers-assq* #"referer" (request-headers/raw req))))
|
||||
(when (or (not origin-header) (not (string-prefix? (bytes->string/latin-1 (header-value origin-header)) (config-get 'canonical_origin))))
|
||||
(return (response/jsexpr
|
||||
#:code 400
|
||||
#:headers always-headers
|
||||
'#hasheq((error .
|
||||
#hasheq((code . "breezewiki")
|
||||
(info . "Origin/Referer header failed validation - cross-origin requests are not allowed here")))))))
|
||||
|
||||
(define post-data/string (bytes->string/utf-8 post-data/bytes))
|
||||
(define post-data (string->jsexpr post-data/string))
|
||||
(define wikiname (jp "/wikiname" post-data))
|
||||
(define path (jp "/path" post-data))
|
||||
(take-json-rewrite-and-return-page
|
||||
#:req req
|
||||
#:wikiname wikiname
|
||||
#:source-url (format "https://~a.fandom.com/wiki/~a" wikiname path)
|
||||
#:data (jp "/data" post-data)
|
||||
#:siteinfo (data->siteinfo (jp "/siteinfo" post-data))))))
|
||||
|
||||
(define (take-json-rewrite-and-return-page #:req req #:wikiname wikiname #:source-url source-url #:data data #:siteinfo siteinfo)
|
||||
(define title (jp "/parse/title" data ""))
|
||||
(define page-html (preprocess-html-wiki (jp "/parse/text" data "")))
|
||||
(define page (html->xexp page-html))
|
||||
(define head-data ((head-data-getter wikiname) data))
|
||||
(response-handler
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
(update-tree-wiki page wikiname)
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title title
|
||||
#:head-data head-data
|
||||
#:siteinfo siteinfo))
|
||||
(define redirect-query-parameter (dict-ref (url-query (request-uri req)) 'redirect "yes"))
|
||||
(define redirect-msg ((query-selector (attribute-selector 'class "redirectMsg") body)))
|
||||
(define redirect-msg-a (if redirect-msg
|
||||
((query-selector (λ (t a c) (eq? t 'a)) redirect-msg))
|
||||
#f))
|
||||
(define html (xexp->html-bytes body))
|
||||
(define headers
|
||||
(build-headers
|
||||
always-headers
|
||||
; redirect-query-parameter: only the string "no" is significant:
|
||||
; https://github.com/Wikia/app/blob/fe60579a53f16816d65dad1644363160a63206a6/includes/Wiki.php#L367
|
||||
(when (and redirect-msg-a
|
||||
(not (equal? redirect-query-parameter "no")))
|
||||
(let* ([dest (get-attribute 'href (bits->attributes redirect-msg-a))]
|
||||
[value (bytes-append #"0;url=" (string->bytes/utf-8 dest))])
|
||||
(header #"Refresh" value)))))
|
||||
(response/full
|
||||
200
|
||||
#"OK"
|
||||
(current-seconds)
|
||||
#"text/html; charset=utf-8"
|
||||
headers
|
||||
(list html))))
|
||||
|
||||
@@ -1,36 +1,52 @@
|
||||
#lang racket/base
|
||||
(require racket/string
|
||||
(require racket/dict
|
||||
racket/string
|
||||
(prefix-in easy: net/http-easy)
|
||||
net/url
|
||||
web-server/http
|
||||
html-writing
|
||||
"application-globals.rkt"
|
||||
"config.rkt"
|
||||
"endpoints.rkt"
|
||||
"fandom-request.rkt"
|
||||
"../lib/url-utils.rkt"
|
||||
"../lib/xexpr-utils.rkt")
|
||||
|
||||
(provide
|
||||
search-fandom)
|
||||
page-search-fandom)
|
||||
|
||||
(module+ test
|
||||
(require rackunit
|
||||
"test-utils.rkt")
|
||||
(define search-results-data
|
||||
'(#hasheq((ns . 0) (pageid . 219) (size . 1482) (snippet . "") (timestamp . "2022-08-21T08:54:23Z") (title . "Gacha Capsule") (wordcount . 214)) #hasheq((ns . 0) (pageid . 201) (size . 1198) (snippet . "") (timestamp . "2022-07-11T17:52:47Z") (title . "Badges") (wordcount . 181)))))
|
||||
|
||||
(define (search-fandom wikiname query params)
|
||||
(define res
|
||||
(fandom-get-api
|
||||
wikiname
|
||||
`(("action" . "query")
|
||||
(define-endpoint
|
||||
search-endpoint
|
||||
[variables
|
||||
(define wikiname (path/param-path (car (url-path (request-uri req)))))
|
||||
(define params (url-query (request-uri req)))
|
||||
(define query (dict-ref params 'q #f))
|
||||
(define title "Search")
|
||||
(define path (format "Special:Search?~a" (params->query `(("query" . ,query)
|
||||
("search" . "internal")))))
|
||||
(define source-url (format "https://~a.fandom.com/wiki/~a" wikiname path))]
|
||||
[endpoints
|
||||
(search
|
||||
(("action" . "query")
|
||||
("list" . "search")
|
||||
("srsearch" . ,query)
|
||||
("formatversion" . "2")
|
||||
("format" . "json"))))
|
||||
(define json (easy:response-json res))
|
||||
(define search-results (jp "/query/search" json))
|
||||
(generate-results-content-fandom wikiname query search-results))
|
||||
|
||||
;;; generate content for display in the wiki page layout
|
||||
(define (generate-results-content-fandom wikiname query search-results)
|
||||
("format" . "json")))
|
||||
(siteinfo
|
||||
(("action" . "query")
|
||||
("meta" . "siteinfo")
|
||||
("siprop" . "general|rightsinfo")
|
||||
("format" . "json")
|
||||
("formatversion" . "2")))]
|
||||
[render
|
||||
(define search-results (jp "/query/search" search))
|
||||
(define body
|
||||
(generate-wiki-page
|
||||
#:req req
|
||||
#:source-url source-url
|
||||
#:wikiname wikiname
|
||||
#:title query
|
||||
#:siteinfo siteinfo
|
||||
`(div (@ (class "mw-parser-output"))
|
||||
;; header before the search results showing how many we found
|
||||
(p ,(format "~a results found for " (length search-results))
|
||||
@@ -51,9 +67,29 @@
|
||||
(time (@ (datetime ,timestamp)) ,(list-ref (string-split timestamp "T") 0))
|
||||
,(format ", ~a words, ~a kb"
|
||||
wordcount
|
||||
(exact->inexact (/ (round (/ size 100)) 10))))))))))
|
||||
(exact->inexact (/ (round (/ size 100)) 10)))))))))))
|
||||
(when (config-true? 'debug)
|
||||
(xexp->html body))
|
||||
(response/output
|
||||
#:code 200
|
||||
#:headers (build-headers always-headers)
|
||||
(λ (out)
|
||||
(write-html body out)))])
|
||||
|
||||
(module+ test
|
||||
(parameterize ([(config-parameter 'feature_offline::only) "false"])
|
||||
(check-not-false ((query-selector (attribute-selector 'href "/test/wiki/Gacha_Capsule")
|
||||
(generate-results-content-fandom "test" "Gacha" search-results-data))))))
|
||||
|
||||
(define-standard-handler (page-search-standard req)
|
||||
#'search-endpoint)
|
||||
|
||||
(define-jsonp-handler (page-search-jsonp req)
|
||||
#'search-endpoint)
|
||||
|
||||
(define-post-data-handler (page-search-with-data req)
|
||||
#'search-endpoint
|
||||
(define params (url-query (request-uri req)))
|
||||
(define query (dict-ref params 'q #f))
|
||||
(define title "Search"))
|
||||
|
||||
(define page-search-fandom
|
||||
(make-switch-handler #:standard page-search-standard
|
||||
#:jsonp page-search-jsonp
|
||||
#:post page-search-with-data))
|
||||
|
||||
@@ -37,23 +37,29 @@ window.proxy = new Proxy(jsonpData, {
|
||||
// *** Data upload and download
|
||||
|
||||
async function cont() {
|
||||
if (jsonpData.wikipage?.error) return error(jsonpData.wikipage)
|
||||
if (jsonpData.siteinfo?.error) return error(jsonpData.siteinfo)
|
||||
if (!(jsonpData.wikipage && jsonpData.siteinfo)) return
|
||||
// Check for errors
|
||||
for (const v of Object.values(jsonpData)) {
|
||||
if (v.error) return error(v)
|
||||
}
|
||||
|
||||
// Check for completion
|
||||
const dependencies = [...document.querySelectorAll("[data-jsonp-var]")].map(e => e.getAttribute("data-jsonp-var"))
|
||||
for (const d of dependencies) {
|
||||
if (!(d in jsonpData)) return
|
||||
}
|
||||
|
||||
const xhr = new XMLHttpRequest();
|
||||
|
||||
const uploadFraction = 0.7
|
||||
|
||||
const pkg = {
|
||||
url: "/api/render/wiki",
|
||||
url: location.href,
|
||||
init: {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
data: jsonpData.wikipage,
|
||||
siteinfo: jsonpData.siteinfo,
|
||||
wikiname: BWData.wikiname,
|
||||
path: BWData.path
|
||||
path: BWData.path,
|
||||
...jsonpData
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -127,7 +133,7 @@ function error(data) {
|
||||
if (typeof data === "string") {
|
||||
render(html`<p><strong>BreezeWiki ran into an error on this page.</strong></p><p>Try reloading the page.</p><p>If this keeps happening, you could <a href="mailto:~cadence/breezewiki-discuss@lists.sr.ht">send a public bug report</a>. Please include the following information:</p><pre>URL: ${window.location.href}${"\n"}${data}</pre>`, eContent)
|
||||
} else if (data.error.code === "missingtitle") {
|
||||
render(html`<p><strong>This page doesn't exist on Fandom.</strong></p><p><small><a href="/${BWData.wikiname}/wiki/Main_Page">Return to the homepage?</a></small></p>`, eContent)
|
||||
render(html`<p><strong>This page doesn't exist on Fandom.</strong></p><p><small><a href="/${BWData.wikiname}/wiki/Main_Page">Return to the wiki's main page</a></small></p>`, eContent)
|
||||
} else {
|
||||
render(html`<p>BreezeWiki wasn't able to load this page.</p><p><strong>${data.error.code}: ${data.error.info}</strong></p>`, eContent)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user