Refactor Input Handling

lychee-lib/src/collector.rs

@@ -3,8 +3,8 @@ use crate::InputSource;
 use crate::filter::PathExcludes;
 use crate::types::resolver::UrlContentResolver;
 use crate::{
-    Base, Input, Request, Result, basic_auth::BasicAuthExtractor, extract::Extractor,
-    types::FileExtensions, types::uri::raw::RawUri, utils::request,
+    Base, Input, InputResolver, Request, Result, basic_auth::BasicAuthExtractor,
+    extract::Extractor, types::FileExtensions, types::uri::raw::RawUri, utils::request,
 };
 use dashmap::DashSet;
 use futures::TryStreamExt;
@@ -188,7 +188,8 @@ impl Collector {
                 let excluded_paths = excluded_paths.clone();
                 let file_extensions = file_extensions.clone();
                 async move {
-                    let input_sources = input.get_input_sources(
+                    let input_sources = InputResolver::resolve(
+                        &input,
                         file_extensions,
                         skip_hidden,
                         skip_ignored,

lychee-lib/src/lib.rs

@@ -99,8 +99,8 @@ pub use crate::{
     filter::{Excludes, Filter, Includes},
     types::{
         AcceptRange, AcceptRangeError, Base, BasicAuthCredentials, BasicAuthSelector, CacheStatus,
-        CookieJar, ErrorKind, FileExtensions, FileType, Input, InputContent, InputSource, Request,
-        Response, ResponseBody, Result, Status, StatusCodeExcluder, StatusCodeSelector,
-        uri::valid::Uri,
+        CookieJar, ErrorKind, FileExtensions, FileType, Input, InputContent, InputResolver,
+        InputSource, Request, Response, ResponseBody, Result, Status, StatusCodeExcluder,
+        StatusCodeSelector, uri::valid::Uri,
     },
 };

lychee-lib/src/types/input/content.rs

@@ -0,0 +1,58 @@
+//! Input content representation and construction.
+//!
+//! The `InputContent` type represents the actual content extracted from various
+//! input sources, along with metadata about the source and file type.
+
+use super::source::InputSource;
+use crate::ErrorKind;
+use crate::types::FileType;
+use std::fs;
+use std::path::PathBuf;
+
+/// Encapsulates the content for a given input
+#[derive(Debug)]
+pub struct InputContent {
+    /// Input source
+    pub source: InputSource,
+    /// File type of given input
+    pub file_type: FileType,
+    /// Raw UTF-8 string content
+    pub content: String,
+}
+
+impl InputContent {
+    #[must_use]
+    /// Create an instance of `InputContent` from an input string
+    pub fn from_string(s: &str, file_type: FileType) -> Self {
+        // TODO: consider using Cow (to avoid one .clone() for String types)
+        Self {
+            source: InputSource::String(s.to_owned()),
+            file_type,
+            content: s.to_owned(),
+        }
+    }
+}
+
+impl TryFrom<&PathBuf> for InputContent {
+    type Error = crate::ErrorKind;
+
+    fn try_from(path: &PathBuf) -> std::result::Result<Self, Self::Error> {
+        let input = match fs::read_to_string(path) {
+            Ok(content) => content,
+            Err(e) if e.kind() == std::io::ErrorKind::InvalidData => {
+                log::warn!(
+                    "Skipping file with invalid UTF-8 content: {}",
+                    path.display()
+                );
+                return Err(ErrorKind::ReadFileInput(e, path.clone()));
+            }
+            Err(e) => return Err(ErrorKind::ReadFileInput(e, path.clone())),
+        };
+
+        Ok(Self {
+            source: InputSource::String(input.clone()),
+            file_type: FileType::from(path),
+            content: input,
+        })
+    }
+}