From 94cb80ea810c6ee6ee370ac02afb6924f9e8485c Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 18:21:35 -0800 Subject: [PATCH 01/11] Handle parse errors gracefully on timeline items Prior to this commit, if even a single item fails to parse Invidious will throw out an error. This means that even if everything else on a page can be parsed and rendered without issues, the single problematic item will cause the entire page to be unusable. This commit gracefully handles parse errors by catching and then replacing the problematic item with a new "timeline error" object that represents the parse error. This will allow the rest of the page to be rendered and an error card that will replace the location of the problematic item. --- src/invidious/helpers/serialized_yt_data.cr | 20 +++++- src/invidious/views/components/item.ecr | 6 +- src/invidious/yt_backend/extractors.cr | 73 ++++++++++++++++----- 3 files changed, 82 insertions(+), 17 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index f8e8f1871..ff2336192 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -291,6 +291,24 @@ struct SearchHashtag end end +# A `ProblematicTimelineItem` is a `SearchItem` created by Invidious that +# represents an item that caused an exception during parsing. +# +# This is not a parsed object from YouTube but rather an Invidious-only type +# created to gracefully communicate parse errors without throwing away +# the rest of the (hopefully) successfully parsed item on a page. +struct ProblematicTimelineItem + property parse_exception : Exception + + def initialize(@parse_exception); end + + def to_json(locale : String?, json : JSON::Builder) + json.object do + json.field "type", "parse-error" + end + end +end + class Category include DB::Serializable @@ -333,4 +351,4 @@ struct Continuation end end -alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category +alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | SearchHashtag | Category | ProblematicTimelineItem diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index c966a9264..79cc47251 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -1,6 +1,6 @@ <%- thin_mode = env.get("preferences").as(Preferences).thin_mode - item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category) && env.get?("user").try &.as(User).watched.index(item.id) != nil + item_watched = !item.is_a?(SearchChannel | SearchHashtag | SearchPlaylist | InvidiousPlaylist | Category | ProblematicTimelineItem) && env.get?("user").try &.as(User).watched.index(item.id) != nil author_verified = item.responds_to?(:author_verified) && item.author_verified -%> @@ -97,6 +97,10 @@ <% when Category %> + <% when ProblematicTimelineItem %> +
+

Unable to parse this item

+
<% else %> <%- # `endpoint_params` is used for the "video-context-buttons" component diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index edd7bf1bb..11ab74837 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -35,6 +35,20 @@ record AuthorFallback, name : String, id : String # data is passed to the private `#parse()` method which returns a datastruct of the given # type. Otherwise, nil is returned. private module Parsers + module BaseParser + def parse(*args) + begin + return parse_internal(*args) + rescue ex + LOGGER.debug("#{ {{@type.name}} }: Failed to render item.") + LOGGER.debug("#{ {{@type.name}} }: Got exception: #{ex.message}") + ProblematicTimelineItem.new( + parse_exception: ex + ) + end + end + end + # Parses a InnerTube videoRenderer into a SearchVideo. Returns nil when the given object isn't a videoRenderer # # A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not** @@ -45,13 +59,16 @@ private module Parsers # `videoRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. # module VideoRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?) return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) video_id = item_contents["videoId"].as_s title = extract_text(item_contents["title"]?) || "" @@ -170,13 +187,16 @@ private module Parsers # `channelRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. # module ChannelRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?) return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) author = extract_text(item_contents["title"]) || author_fallback.name author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id author_verified = has_verified_badge?(item_contents["ownerBadges"]?) @@ -230,13 +250,16 @@ private module Parsers # A `hashtagTileRenderer` is a kind of search result. # It can be found when searching for any hashtag (e.g "#hi" or "#shorts") module HashtagRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["hashtagTileRenderer"]? return self.parse(item_contents) end end - private def self.parse(item_contents) + private def parse_internal(item_contents) title = extract_text(item_contents["hashtag"]).not_nil! # E.g "#hi" # E.g "/hashtag/hi" @@ -263,10 +286,6 @@ private module Parsers video_count: short_text_to_number(video_count_txt || ""), channel_count: short_text_to_number(channel_count_txt || ""), }) - rescue ex - LOGGER.debug("HashtagRendererParser: Failed to extract renderer.") - LOGGER.debug("HashtagRendererParser: Got exception: #{ex.message}") - return nil end def self.parser_name @@ -284,13 +303,16 @@ private module Parsers # `gridPlaylistRenderer`s can be found on the playlist-tabs of channels and expanded categories. # module GridPlaylistRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["gridPlaylistRenderer"]? return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) title = extract_text(item_contents["title"]) || "" plid = item_contents["playlistId"]?.try &.as_s || "" @@ -325,13 +347,16 @@ private module Parsers # `playlistRenderer`s can be found almost everywhere on YouTube. In categories, search results, recommended, etc. # module PlaylistRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["playlistRenderer"]? return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) title = extract_text(item_contents["title"]) || "" plid = item_contents["playlistId"]?.try &.as_s || "" @@ -385,13 +410,16 @@ private module Parsers # `shelfRenderer`s can be found almost everywhere on YouTube. In categories, search results, channels, etc. # module CategoryRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["shelfRenderer"]? return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) title = extract_text(item_contents["title"]?) || "" url = item_contents.dig?("endpoint", "commandMetadata", "webCommandMetadata", "url") .try &.as_s @@ -450,13 +478,16 @@ private module Parsers # container.It is very similar to RichItemRendererParser # module ItemSectionRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item.dig?("itemSectionRenderer", "contents", 0) return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) child = VideoRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) @@ -476,13 +507,16 @@ private module Parsers # itself inside a richGridRenderer container. # module RichItemRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item.dig?("richItemRenderer", "content") return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) child = VideoRendererParser.process(item_contents, author_fallback) child ||= ReelItemRendererParser.process(item_contents, author_fallback) child ||= PlaylistRendererParser.process(item_contents, author_fallback) @@ -506,13 +540,16 @@ private module Parsers # TODO: Confirm that hypothesis # module ReelItemRendererParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) video_id = item_contents["videoId"].as_s reel_player_overlay = item_contents.dig( @@ -600,13 +637,16 @@ private module Parsers # a richItemRenderer or a richGridRenderer. # module LockupViewModelParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["lockupViewModel"]? return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) playlist_id = item_contents["contentId"].as_s thumbnail_view_model = item_contents.dig( @@ -675,13 +715,16 @@ private module Parsers # usually (always?) encapsulated in a richItemRenderer. # module ShortsLockupViewModelParser + extend self + include BaseParser + def self.process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["shortsLockupViewModel"]? return self.parse(item_contents, author_fallback) end end - private def self.parse(item_contents, author_fallback) + private def parse_internal(item_contents, author_fallback) # TODO: Maybe add support for "oardefault.jpg" thumbnails? # thumbnail = item_contents.dig("thumbnail", "sources", 0, "url").as_s # Gives: https://i.ytimg.com/vi/{video_id}/oardefault.jpg?... From dbeee714577846e496eedbf4fb18cf20c66115ea Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 19:47:01 -0800 Subject: [PATCH 02/11] Apply search filters details css only to itself The CSS for the search filters details box was applied to every detail element when search.css is loaded --- assets/css/search.css | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/assets/css/search.css b/assets/css/search.css index 7036fd285..833ec7e90 100644 --- a/assets/css/search.css +++ b/assets/css/search.css @@ -1,4 +1,4 @@ -summary { +#filters-collapse summary { /* This should hide the marker */ display: block; @@ -8,10 +8,10 @@ summary { cursor: pointer; } -summary::-webkit-details-marker, -summary::marker { display: none; } +#filters-collapse summary::-webkit-details-marker, +#filters-collapse summary::marker { display: none; } -summary:before { +#filters-collapse summary:before { border-radius: 5px; content: "[ + ]"; margin: -2px 10px 0 10px; @@ -20,7 +20,7 @@ summary:before { width: 40px; } -details[open] > summary:before { content: "[ − ]"; } +#filters-collapse details[open] > summary:before { content: "[ − ]"; } #filters-box { From 9de69c0052b004c22a1d83f10193e13d1f3d5c58 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 20:42:07 -0800 Subject: [PATCH 03/11] Improve design of placeholder item Also makes it show the error backtrace --- assets/css/default.css | 47 +++++++++++++++++++++++++ locales/en-US.json | 4 ++- src/invidious/helpers/errors.cr | 29 +++++++++------ src/invidious/views/components/item.ecr | 12 +++++-- 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/assets/css/default.css b/assets/css/default.css index 2cedcf0c5..8de224eb5 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -816,3 +816,50 @@ h1, h2, h3, h4, h5, p, #download_widget { width: 100%; } + +.error-card { + display: flex; + flex-direction: column; + align-items: center; + padding: 25px; + margin-bottom: 1em; + border-radius: 10px; + border: 1px solid black; + box-sizing: border-box; + height: 100%; +} + +.error-card > .explanation { + display: grid; + grid-template-columns: max-content 1fr; + grid-template-rows: 1fr max-content; + column-gap: 10px; + row-gap: 4px; +} + +.error-card > .explanation > i { + grid-area: 1 / 1 / 2 / 2; +} + +.error-card > .explanation > h4 { + grid-area: 1 / 2 / 2 / 3; + margin: 0; +} + +.error-card > .explanation > p { + grid-area: 2 / 2 / 3 / 3; + margin: 0; +} + +.error-card details { + margin-top: 10px; + width: 100%; +} + +.error-card summary { + width: 100%; +} + +.error-card pre { + height: 300px; +} \ No newline at end of file diff --git a/locales/en-US.json b/locales/en-US.json index 4f2c27703..c9a489728 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -501,5 +501,7 @@ "toggle_theme": "Toggle Theme", "carousel_slide": "Slide {{current}} of {{total}}", "carousel_skip": "Skip the Carousel", - "carousel_go_to": "Go to slide `x`" + "carousel_go_to": "Go to slide `x`", + "timeline_parse_error_placeholder_heading": "Unable to parse item", + "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:" } diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 900cb0c63..399324cdb 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -18,16 +18,7 @@ def github_details(summary : String, content : String) return HTML.escape(details) end -def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) - if exception.is_a?(InfoException) - return error_template_helper(env, status_code, exception.message || "") - end - - locale = env.get("preferences").as(Preferences).locale - - env.response.content_type = "text/html" - env.response.status_code = status_code - +def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tuple(String, String) issue_title = "#{exception.message} (#{exception.class})" issue_template = <<-TEXT @@ -40,6 +31,24 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce issue_template += github_details("Backtrace", exception.inspect_with_backtrace) + return {issue_title, issue_template} +end + +def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) + if exception.is_a?(InfoException) + return error_template_helper(env, status_code, exception.message || "") + end + + locale = env.get("preferences").as(Preferences).locale + + env.response.content_type = "text/html" + env.response.status_code = status_code + + # Unpacking into issue_title, issue_template directly causes a compiler error + # I have no idea why. + issue_template_components = get_issue_template(env, exception) + issue_title, issue_template = issue_template_components + # URLs for the error message below url_faq = "https://github.com/iv-org/documentation/blob/master/docs/faq.md" url_search_issues = "https://github.com/iv-org/invidious/issues" diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 79cc47251..348ea127f 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -98,8 +98,16 @@ <% when Category %> <% when ProblematicTimelineItem %> -
-

Unable to parse this item

+
+
+ +

<%=translate(locale, "timeline_parse_error_placeholder_heading")%>

+

<%=translate(locale, "timeline_parse_error_placeholder_message")%>

+
+
+ Show technical details +
<%=get_issue_template(env, item.parse_exception)[1]%>
+
<% else %> <%- From 0e0a95430a19130304bc9f4cf8e766e8783865b0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 20:42:37 -0800 Subject: [PATCH 04/11] Improve JSON repr of ProblematicTimelineItem --- src/invidious/helpers/serialized_yt_data.cr | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index ff2336192..5eef23599 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -305,6 +305,8 @@ struct ProblematicTimelineItem def to_json(locale : String?, json : JSON::Builder) json.object do json.field "type", "parse-error" + json.field "errorMessage", @parse_exception.message + json.field "errorBacktrace", @parse_exception.inspect_with_backtrace end end end From 180d77276b92f8a54946dd1c4252e07fea12dc47 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 20:58:16 -0800 Subject: [PATCH 05/11] Emphasise error card icon --- assets/css/default.css | 3 +++ 1 file changed, 3 insertions(+) diff --git a/assets/css/default.css b/assets/css/default.css index 8de224eb5..15eee34fa 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -833,11 +833,14 @@ h1, h2, h3, h4, h5, p, display: grid; grid-template-columns: max-content 1fr; grid-template-rows: 1fr max-content; + align-items: center; column-gap: 10px; row-gap: 4px; } .error-card > .explanation > i { + color: #f44; + font-size: 24px; grid-area: 1 / 1 / 2 / 2; } From dd16f15aaeb2b1110fa9c26f8f70c494e0e8b6e6 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 21:01:26 -0800 Subject: [PATCH 06/11] Improve error card border color on dark theme --- assets/css/default.css | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/assets/css/default.css b/assets/css/default.css index 15eee34fa..9d2ab34b5 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -550,6 +550,10 @@ span > select { color: #565d64; } +.light-theme .error-card { + border: 1px solid black; +} + @media (prefers-color-scheme: light) { .no-theme a:hover, .no-theme a:active, @@ -596,6 +600,10 @@ span > select { .light-theme .pure-menu-heading { color: #565d64; } + + .no-theme .error-card { + border: 1px solid black; + } } @@ -658,6 +666,10 @@ body.dark-theme { color: inherit; } +.dark-theme .error-card { + border: 1px solid #5e5e5e; +} + @media (prefers-color-scheme: dark) { .no-theme a:hover, .no-theme a:active, @@ -719,6 +731,10 @@ body.dark-theme { .no-theme footer a { color: #adadad !important; } + + .no-theme .error-card { + border: 1px solid #5e5e5e; + } } @@ -824,7 +840,6 @@ h1, h2, h3, h4, h5, p, padding: 25px; margin-bottom: 1em; border-radius: 10px; - border: 1px solid black; box-sizing: border-box; height: 100%; } From aae5ba01c2355ef9b67c95f0227fb47ad89eacf4 Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 21:02:14 -0800 Subject: [PATCH 07/11] Fix formatting --- src/invidious/yt_backend/extractors.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 11ab74837..321957f1c 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -40,8 +40,8 @@ private module Parsers begin return parse_internal(*args) rescue ex - LOGGER.debug("#{ {{@type.name}} }: Failed to render item.") - LOGGER.debug("#{ {{@type.name}} }: Got exception: #{ex.message}") + LOGGER.debug("#{{{@type.name}}}: Failed to render item.") + LOGGER.debug("#{{{@type.name}}}: Got exception: #{ex.message}") ProblematicTimelineItem.new( parse_exception: ex ) From c288005bfd512652f82d3bfa8b2ff939dbd694de Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 28 Feb 2025 21:05:19 -0800 Subject: [PATCH 08/11] Make "show technical details" btn translatable --- locales/en-US.json | 3 ++- src/invidious/views/components/item.ecr | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/locales/en-US.json b/locales/en-US.json index c9a489728..d44a88273 100644 --- a/locales/en-US.json +++ b/locales/en-US.json @@ -503,5 +503,6 @@ "carousel_skip": "Skip the Carousel", "carousel_go_to": "Go to slide `x`", "timeline_parse_error_placeholder_heading": "Unable to parse item", - "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:" + "timeline_parse_error_placeholder_message": "Invidious encountered an error while trying to parse this item. For more information see below:", + "timeline_parse_error_show_technical_details": "Show technical details" } diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 348ea127f..279a74b23 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -105,7 +105,7 @@

<%=translate(locale, "timeline_parse_error_placeholder_message")%>

- Show technical details + <%=translate(locale, "timeline_parse_error_show_technical_details")%>
<%=get_issue_template(env, item.parse_exception)[1]%>
From f7810ba007d088175c693648d4e040a9fe1101a4 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 19 Mar 2025 23:32:46 -0700 Subject: [PATCH 09/11] Use ProblematicTimelineItem as needed in playlists --- src/invidious/helpers/serialized_yt_data.cr | 26 +++++++++++++++++++++ src/invidious/playlists.cr | 8 +++++-- src/invidious/routes/embed.cr | 8 +++++-- src/invidious/routes/feeds.cr | 8 ++++++- 4 files changed, 45 insertions(+), 5 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 5eef23599..56b64dbf7 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -309,6 +309,32 @@ struct ProblematicTimelineItem json.field "errorBacktrace", @parse_exception.inspect_with_backtrace end end + + # Provides compatibility with PlaylistVideo + def to_json(json : JSON::Builder, *args, **kwargs) + return to_json("", json) + end + + def to_xml(env, locale, xml : XML::Builder) + xml.element("entry") do + xml.element("id") { xml.text "iv-err-#{Random.new.base64(8)}" } + xml.element("title") { xml.text "Parse Error: This item has failed to parse" } + xml.element("updated") { xml.text Time.utc.to_rfc3339 } + + xml.element("content", type: "xhtml") do + xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do + xml.element("div") do + xml.element("h4") { translate(locale, "timeline_parse_error_placeholder_heading") } + xml.element("p") { translate(locale, "timeline_parse_error_placeholder_message") } + end + + xml.element("pre") do + get_issue_template(env, @parse_exception) + end + end + end + end + end end class Category diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index b670c0093..c762b64b2 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -432,7 +432,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, offset = initial_data.dig?("contents", "twoColumnWatchNextResults", "playlist", "playlist", "currentIndex").try &.as_i || offset end - videos = [] of PlaylistVideo + videos = [] of PlaylistVideo | ProblematicTimelineItem until videos.size >= 200 || videos.size == playlist.video_count || offset >= playlist.video_count # 100 videos per request @@ -448,7 +448,7 @@ def get_playlist_videos(playlist : InvidiousPlaylist | Playlist, offset : Int32, end def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) - videos = [] of PlaylistVideo + videos = [] of PlaylistVideo | ProblematicTimelineItem if initial_data["contents"]? tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] @@ -500,6 +500,10 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) index: index, }) end + rescue ex + videos << ProblematicTimelineItem.new( + parse_exception: ex + ) end return videos diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index bdbb2d895..b7ae4e0ec 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -12,13 +12,15 @@ module Invidious::Routes::Embed url = "/playlist?list=#{plid}" raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url)) end + + get_first_video = videos[0].as(PlaylistVideo) rescue ex : NotFoundException return error_template(404, ex) rescue ex return error_template(500, ex) end - url = "/embed/#{videos[0].id}?#{env.params.query}" + url = "/embed/#{get_first_video}?#{env.params.query}" if env.params.query.size > 0 url += "?#{env.params.query}" @@ -72,13 +74,15 @@ module Invidious::Routes::Embed url = "/playlist?list=#{plid}" raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url)) end + + get_first_video = videos[0].as(PlaylistVideo) rescue ex : NotFoundException return error_template(404, ex) rescue ex return error_template(500, ex) end - url = "/embed/#{videos[0].id}" + url = "/embed/#{get_first_video.id}" elsif video_series url = "/embed/#{video_series.shift}" env.params.query["playlist"] = video_series.join(",") diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr index 7f9a0edbc..abfea9ee7 100644 --- a/src/invidious/routes/feeds.cr +++ b/src/invidious/routes/feeds.cr @@ -296,7 +296,13 @@ module Invidious::Routes::Feeds xml.element("name") { xml.text playlist.author } end - videos.each &.to_xml(xml) + videos.each do |video| + if video.is_a? PlaylistVideo + video.to_xml(xml) + else + video.to_xml(env, locale, xml) + end + end end end else From 7b2758545429e5ad09f0182bf71b351a52815a1d Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 19 Mar 2025 23:50:41 -0700 Subject: [PATCH 10/11] Support ProblematicTimelineItem in trending feed --- src/invidious/helpers/serialized_yt_data.cr | 5 ++++- src/invidious/trending.cr | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 56b64dbf7..a6501e413 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -299,8 +299,11 @@ end # the rest of the (hopefully) successfully parsed item on a page. struct ProblematicTimelineItem property parse_exception : Exception + property id : String - def initialize(@parse_exception); end + def initialize(@parse_exception) + @id = Random.new.hex(8) + end def to_json(locale : String?, json : JSON::Builder) json.object do diff --git a/src/invidious/trending.cr b/src/invidious/trending.cr index 107d148d0..d14cde5d9 100644 --- a/src/invidious/trending.cr +++ b/src/invidious/trending.cr @@ -31,12 +31,12 @@ def fetch_trending(trending_type, region, locale) # See: https://github.com/iv-org/invidious/issues/2989 next if (itm.contents.size < 24 && deduplicate) - extracted.concat extract_category(itm) + extracted.concat itm.contents.select(SearchItem) else extracted << itm end end # Deduplicate items before returning results - return extracted.select(SearchVideo).uniq!(&.id), plid + return extracted.select(SearchVideo | ProblematicTimelineItem).uniq!(&.id), plid end From 6c063436d4259d62ed6794e99298348b0be74b61 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 5 Apr 2025 12:23:42 -0700 Subject: [PATCH 11/11] Fix issues raised by code review Remove explicit `self.` from #process of parsers Remove explicit return tuple in get_issue_template Fix formatting Move inline issue template style to stylesheet Use @id in ProblematicTimelineItem xml repr Fix naming --- assets/css/default.css | 5 +++++ src/invidious/helpers/errors.cr | 4 ++-- src/invidious/helpers/serialized_yt_data.cr | 2 +- src/invidious/playlists.cr | 4 +--- src/invidious/routes/embed.cr | 8 ++++---- src/invidious/views/components/item.ecr | 2 +- src/invidious/yt_backend/extractors.cr | 22 ++++++++++----------- 7 files changed, 25 insertions(+), 22 deletions(-) diff --git a/assets/css/default.css b/assets/css/default.css index 9d2ab34b5..01d4b736c 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -880,4 +880,9 @@ h1, h2, h3, h4, h5, p, .error-card pre { height: 300px; +} + +.error-issue-template { + padding: 20px; + background: rgba(0, 0, 0, 0.12345); } \ No newline at end of file diff --git a/src/invidious/helpers/errors.cr b/src/invidious/helpers/errors.cr index 399324cdb..e2c4b650a 100644 --- a/src/invidious/helpers/errors.cr +++ b/src/invidious/helpers/errors.cr @@ -31,7 +31,7 @@ def get_issue_template(env : HTTP::Server::Context, exception : Exception) : Tup issue_template += github_details("Backtrace", exception.inspect_with_backtrace) - return {issue_title, issue_template} + return issue_title, issue_template end def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception) @@ -78,7 +78,7 @@ def error_template_helper(env : HTTP::Server::Context, status_code : Int32, exce

#{translate(locale, "crash_page_report_issue", url_new_issue)}

-
#{issue_template}
+
#{issue_template}
END_HTML diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index a6501e413..2796a8dc6 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -320,7 +320,7 @@ struct ProblematicTimelineItem def to_xml(env, locale, xml : XML::Builder) xml.element("entry") do - xml.element("id") { xml.text "iv-err-#{Random.new.base64(8)}" } + xml.element("id") { xml.text "iv-err-#{@id}" } xml.element("title") { xml.text "Parse Error: This item has failed to parse" } xml.element("updated") { xml.text Time.utc.to_rfc3339 } diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index c762b64b2..7c584d153 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -501,9 +501,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) }) end rescue ex - videos << ProblematicTimelineItem.new( - parse_exception: ex - ) + videos << ProblematicTimelineItem.new(parse_exception: ex) end return videos diff --git a/src/invidious/routes/embed.cr b/src/invidious/routes/embed.cr index b7ae4e0ec..930e4915a 100644 --- a/src/invidious/routes/embed.cr +++ b/src/invidious/routes/embed.cr @@ -13,14 +13,14 @@ module Invidious::Routes::Embed raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url)) end - get_first_video = videos[0].as(PlaylistVideo) + first_playlist_video = videos[0].as(PlaylistVideo) rescue ex : NotFoundException return error_template(404, ex) rescue ex return error_template(500, ex) end - url = "/embed/#{get_first_video}?#{env.params.query}" + url = "/embed/#{first_playlist_video}?#{env.params.query}" if env.params.query.size > 0 url += "?#{env.params.query}" @@ -75,14 +75,14 @@ module Invidious::Routes::Embed raise NotFoundException.new(translate(locale, "error_video_not_in_playlist", url)) end - get_first_video = videos[0].as(PlaylistVideo) + first_playlist_video = videos[0].as(PlaylistVideo) rescue ex : NotFoundException return error_template(404, ex) rescue ex return error_template(500, ex) end - url = "/embed/#{get_first_video.id}" + url = "/embed/#{first_playlist_video.id}" elsif video_series url = "/embed/#{video_series.shift}" env.params.query["playlist"] = video_series.join(",") diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index 279a74b23..a24423df9 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -106,7 +106,7 @@
<%=translate(locale, "timeline_parse_error_show_technical_details")%> -
<%=get_issue_template(env, item.parse_exception)[1]%>
+
<%=get_issue_template(env, item.parse_exception)[1]%>
<% else %> diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 321957f1c..df2de81d2 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -62,7 +62,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?) return self.parse(item_contents, author_fallback) end @@ -190,7 +190,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?) return self.parse(item_contents, author_fallback) end @@ -253,7 +253,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["hashtagTileRenderer"]? return self.parse(item_contents) end @@ -306,7 +306,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["gridPlaylistRenderer"]? return self.parse(item_contents, author_fallback) end @@ -350,7 +350,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["playlistRenderer"]? return self.parse(item_contents, author_fallback) end @@ -413,7 +413,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["shelfRenderer"]? return self.parse(item_contents, author_fallback) end @@ -481,7 +481,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item.dig?("itemSectionRenderer", "contents", 0) return self.parse(item_contents, author_fallback) end @@ -510,7 +510,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item.dig?("richItemRenderer", "content") return self.parse(item_contents, author_fallback) end @@ -543,7 +543,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["reelItemRenderer"]? return self.parse(item_contents, author_fallback) end @@ -640,7 +640,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["lockupViewModel"]? return self.parse(item_contents, author_fallback) end @@ -718,7 +718,7 @@ private module Parsers extend self include BaseParser - def self.process(item : JSON::Any, author_fallback : AuthorFallback) + def process(item : JSON::Any, author_fallback : AuthorFallback) if item_contents = item["shortsLockupViewModel"]? return self.parse(item_contents, author_fallback) end