feat: Added Example.

.dockerignore

@@ -26,7 +26,7 @@ wheels/
 MANIFEST
 
 # PyInstaller
-#  Usually these files are written by a python script from a example
+#  Usually these files are written by a python script from a comics
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 *.manifest
 *.spec

README.md

@@ -28,7 +28,7 @@ For example:
    {
       "num": {{num}},
       "date": "2023-10-12", 
-      "title": "A simple example"
+      "title": "A simple comics"
    }
    ```
    Then you would design the table class as

cloudquery.log

@@ -0,0 +1,34 @@
+2023-10-20T10:26:35Z INF Loading spec(s) args=["TestConfig.yaml"] module=cli
+2023-10-20T10:26:35Z INF Plugin server listening address="C:\\Users\\plain\\AppData\\Local\\Temp\\cq-fSaZbARejzSPvyWu.sock" module=cli
+2023-10-20T10:26:36Z INF started call grpc.component=server grpc.method=GetVersions grpc.method_type=unary grpc.service=cloudquery.discovery.v1.Discovery grpc.start_time=2023-10-20T11:26:36+01:00 grpc.time_ms=0.514 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:26:36Z INF finished call grpc.code=OK grpc.component=server grpc.method=GetVersions grpc.method_type=unary grpc.service=cloudquery.discovery.v1.Discovery grpc.start_time=2023-10-20T11:26:36+01:00 grpc.time_ms=1.041 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:26:36Z INF Start sync destinations=["sqlite (v2.4.11)"] module=cli source="comics (grpc@localhost:7777)" sync_time=2023-10-20T10:26:36Z
+2023-10-20T10:26:36Z INF started call grpc.component=server grpc.method=Init grpc.method_type=unary grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:26:36+01:00 grpc.time_ms=0 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:26:36Z INF finished call grpc.code=OK grpc.component=server grpc.method=Init grpc.method_type=unary grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:26:36+01:00 grpc.time_ms=0.519 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:26:36Z INF Start fetching resources destinations=["sqlite (v2.4.11)"] module=cli source="comics (grpc@localhost:7777)"
+2023-10-20T10:26:36Z INF started call grpc.component=server grpc.method=Write grpc.method_type=client_stream grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:26:36+01:00 grpc.time_ms=5.194 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:26:36Z INF Migrating table module=sqlite-dest table=cq_comics
+2023-10-20T10:26:58Z INF End sync destinations=["sqlite (v2.4.11)"] module=cli source="comics (grpc@localhost:7777)" sync_time=2023-10-20T10:26:36Z
+2023-10-20T10:26:58Z INF Sending sync summary to analyticsv1.cloudquery.io:443 module=cli
+2023-10-20T10:26:59Z ERR failed to kill source plugin error="TerminateProcess: Access is denied." module=cli
+2023-10-20T10:26:59Z INF waiting for source plugin to terminate module=cli
+2023-10-20T10:26:59Z INF source plugin process exited with exit status 1 module=cli
+2023-10-20T10:26:59Z ERR exiting with error error="received interrupt signal from OS: failed to sync v3 source comics: unexpected error from sync client receive: rpc error: code = Canceled desc = context canceled" module=cli
+2023-10-20T10:27:50Z INF Loading spec(s) args=["TestConfig.yaml"] module=cli
+2023-10-20T10:27:50Z INF Plugin server listening address="C:\\Users\\plain\\AppData\\Local\\Temp\\cq-VawldnMMdSdHjskd.sock" module=cli
+2023-10-20T10:27:51Z INF started call grpc.component=server grpc.method=GetVersions grpc.method_type=unary grpc.service=cloudquery.discovery.v1.Discovery grpc.start_time=2023-10-20T11:27:51+01:00 grpc.time_ms=0.505 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:27:51Z INF finished call grpc.code=OK grpc.component=server grpc.method=GetVersions grpc.method_type=unary grpc.service=cloudquery.discovery.v1.Discovery grpc.start_time=2023-10-20T11:27:51+01:00 grpc.time_ms=0.505 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:27:51Z INF Start sync destinations=["sqlite (v2.4.11)"] module=cli source="comics (grpc@localhost:7777)" sync_time=2023-10-20T10:27:51Z
+2023-10-20T10:27:51Z INF started call grpc.component=server grpc.method=Init grpc.method_type=unary grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:27:51+01:00 grpc.time_ms=0 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:27:51Z INF finished call grpc.code=OK grpc.component=server grpc.method=Init grpc.method_type=unary grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:27:51+01:00 grpc.time_ms=0 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:27:51Z INF Start fetching resources destinations=["sqlite (v2.4.11)"] module=cli source="comics (grpc@localhost:7777)"
+2023-10-20T10:27:51Z INF started call grpc.component=server grpc.method=Write grpc.method_type=client_stream grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:27:51+01:00 grpc.time_ms=1.075 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:27:51Z INF Migrating table module=sqlite-dest table=cq_comics
+2023-10-20T10:27:51Z INF Table exists, auto-migrating module=sqlite-dest table=cq_comics
+2023-10-20T10:32:25Z INF finished call grpc.code=OK grpc.component=server grpc.method=Write grpc.method_type=client_stream grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:27:51+01:00 grpc.time_ms=273715.6 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:32:25Z INF started call grpc.component=server grpc.method=Close grpc.method_type=unary grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:32:25+01:00 grpc.time_ms=0 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:32:25Z INF finished call grpc.code=OK grpc.component=server grpc.method=Close grpc.method_type=unary grpc.service=cloudquery.plugin.v3.Plugin grpc.start_time=2023-10-20T11:32:25+01:00 grpc.time_ms=0.518 module=cli peer.address=@ protocol=grpc
+2023-10-20T10:32:25Z INF End sync destinations=["sqlite (v2.4.11)"] module=cli source="comics (grpc@localhost:7777)" sync_time=2023-10-20T10:27:51Z
+2023-10-20T10:32:25Z INF Sending sync summary to analyticsv1.cloudquery.io:443 module=cli
+2023-10-20T10:32:25Z INF waiting for source plugin to terminate module=cli
+2023-10-20T10:32:25Z INF source plugin process exited with exit status 1 module=cli

main.py

@@ -1,11 +1,11 @@
 import sys
 from cloudquery.sdk import serve
 
-from plugin import ExamplePlugin
+from plugin import XKCDPlugin
 
 
 def main():
-    p = ExamplePlugin()
+    p = XKCDPlugin()
     serve.PluginCommand(p).run(sys.argv[1:])
 
 

plugin/__init__.py

@@ -1 +1 @@
-from .plugin import ExamplePlugin
+from .plugin import XKCDPlugin

plugin/client/client.py

@@ -1,16 +1,15 @@
 from dataclasses import dataclass, field
 from cloudquery.sdk.scheduler import Client as ClientABC
 
-from plugin.example.client import ExampleClient
+from plugin.comics.client import ComicsClient
 
 DEFAULT_CONCURRENCY = 100
 DEFAULT_QUEUE_SIZE = 10000
 
 
 @dataclass
 class Spec:
-    access_token: str
-    base_url: str = field(default="https://api.example.com")
+    base_url: str = field(default="https://xkcd.com")
     concurrency: int = field(default=DEFAULT_CONCURRENCY)
     queue_size: int = field(default=DEFAULT_QUEUE_SIZE)
 
@@ -23,11 +22,11 @@ def validate(self):
 class Client(ClientABC):
     def __init__(self, spec: Spec) -> None:
         self._spec = spec
-        self._client = ExampleClient(spec.access_token, spec.base_url)
+        self._client = ComicsClient(spec.base_url)
 
     def id(self):
-        return "example"
+        return "xkcd"
 
     @property
-    def client(self) -> ExampleClient:
+    def client(self) -> ComicsClient:
         return self._client

plugin/comics/client.py

@@ -0,0 +1,23 @@
+from typing import Generator, Dict, Any
+from urllib.parse import urljoin
+
+import requests
+
+
+class ComicsClient:
+    def __init__(self, base_url="https://xkcd.com"):
+        self._base_url = base_url
+
+    def comic_iterator(self) -> Generator[Dict[str, Any], None, None]:
+        response = requests.get(urljoin(self._base_url, "info.0.json"))
+        if response.status_code != 200:
+            raise ValueError("Bad HTTP Response")
+        yield response.json()
+
+        num = response.json().get("num", 0)
+        while num > 0:
+            num -= 1
+            response = requests.get(urljoin(urljoin(self._base_url, f"/{num}/"), "info.0.json"))
+            if response.status_code != 200:
+                raise ValueError("Bad HTTP Response")
+            yield response.json()

plugin/plugin.py

@@ -10,11 +10,11 @@
 from plugin import tables
 from plugin.client import Client, Spec
 
-PLUGIN_NAME = "example"
-PLUGIN_VERSION = "1.0.0"  # {x-release-please-version}
+PLUGIN_NAME = "xkcd"
+PLUGIN_VERSION = "0.1.0"  # {x-release-please-version}
 
 
-class ExamplePlugin(plugin.Plugin):
+class XKCDPlugin(plugin.Plugin):
     def __init__(self) -> None:
         super().__init__(PLUGIN_NAME, PLUGIN_VERSION)
         self._spec_json = None
@@ -39,7 +39,7 @@ def init(self, spec_bytes, no_connection: bool = False):
 
     def get_tables(self, options: plugin.TableOptions) -> List[plugin.Table]:
         all_tables: List[plugin.Table] = [
-            tables.Items(),
+            tables.Comics(),
         ]
 
         # set parent table relationships

plugin/tables/__init__.py

@@ -1 +1 @@
-from .items import Items
+from .comics import Comics

plugin/tables/comics.py

@@ -0,0 +1,57 @@
+from typing import Any, Generator
+
+import pyarrow as pa
+from cloudquery.sdk.scheduler import TableResolver
+from cloudquery.sdk.schema import Column
+from cloudquery.sdk.schema import Table
+from cloudquery.sdk.schema.resource import Resource
+
+from plugin.client import Client
+
+
+class Comics(Table):
+    """{
+        "month": "10",
+        "num": 2841,
+        "link": "",
+        "year": "2023",
+        "news": "",
+        "safe_title": "Sign Combo",
+        "transcript": "",
+        "alt": "Speed Limit: 45 MPH / Minimum: 65 MPH",
+        "img": "https://imgs.xkcd.com/comics/sign_combo.png",
+        "title": "Sign Combo",
+        "day": "13"
+    }"""
+    def __init__(self) -> None:
+        super().__init__(
+            name="cq_comics",
+            title="Comics",
+            columns=[
+                Column("num", pa.uint64(), primary_key=True),
+                Column("year", pa.string()),
+                Column("month", pa.string()),
+                Column("day", pa.string()),
+                Column("safe_title", pa.string()),
+                Column("title", pa.string()),
+                Column("news", pa.string()),
+                Column("transcript", pa.string()),
+                Column("alt", pa.string()),
+                Column("img", pa.string()),
+            ],
+        )
+
+    @property
+    def resolver(self):
+        return ComicResolver(table=self)
+
+
+class ComicResolver(TableResolver):
+    def __init__(self, table) -> None:
+        super().__init__(table=table)
+
+    def resolve(
+        self, client: Client, parent_resource: Resource
+    ) -> Generator[Any, None, None]:
+        for comic_response in client.client.comic_iterator():
+            yield comic_response