Merge pull request #4 from cloudquery/feat/upgrade-sdk-v4

feat: Upgrade to SDK v4

Author: adriaandotcom

.github/workflows/test.yaml

@@ -23,7 +23,12 @@ jobs:
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.50.1
+          version: v1.54.2
+      - name: Setup CloudQuery
+        if: github.event_name == 'pull_request'
+        uses: cloudquery/setup-cloudquery@v3
+        with:
+          version: v3.27.1
       - name: Get dependencies
         run: go get -t -d ./...
       - name: Build

.gitignore

@@ -1,4 +1,8 @@
 dist/
 vendor/
 bin/
-cq-source-simple-analytics
+cq-source-simple-analytics
+cloudquery
+cloudquery.log
+.DS_Store
+.idea

.golangci.yml

@@ -80,7 +80,6 @@ linters:
   enable:
     - asciicheck
     - bodyclose
-    - depguard
     - dupl
     - errcheck
     - gocritic

Makefile

@@ -1,15 +1,25 @@
+.PHONY: build
+build:
+	go build
+
 .PHONY: test
 test:
-	go test -timeout 3m ./...
+	go test -short -race -timeout 3m ./...
 
 .PHONY: lint
 lint:
 	@golangci-lint run --timeout 10m --verbose
 
 .PHONY: gen-docs
-gen-docs:
-	rm -rf ./docs/tables/*
-	go run main.go doc ./docs/tables
+gen-docs: build
+	@command -v cloudquery >/dev/null 2>&1 || { \
+		echo "Error: 'cloudquery' command not found. Please install it before running gen-docs."; \
+		echo "You can install it by following the instructions at: https://www.cloudquery.io/docs/quickstart"; \
+		exit 1; \
+	}
+	rm -rf docs/tables
+	cloudquery tables --format markdown --output-dir docs/ test/config.yml
+	mv -vf docs/simple-analytics docs/tables
 
 # All gen targets
 .PHONY: gen

README.md

@@ -3,24 +3,27 @@
 [![test](https://github.com/cloudquery/cq-source-simple-analytics/actions/workflows/test.yaml/badge.svg)](https://github.com/cloudquery/cq-source-simple-analytics/actions/workflows/test.yaml)
 [![Go Report Card](https://goreportcard.com/badge/github.com/cloudquery/cq-source-simple-analytics)](https://goreportcard.com/report/github.com/cloudquery/cq-source-simple-analytics)
 
-A [Simple Analytics](https://simpleanalytics.com/) source plugin for CloudQuery that loads raw page view and event data from Simple Analytics to any database, data warehouse or data lake supported by [CloudQuery](https://www.cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
+A [Simple Analytics](https://simpleanalytics.com/) source plugin for CloudQuery that loads raw page view and event data from Simple Analytics to any database, data warehouse or data lake supported by [CloudQuery](https://cloudquery.io/), such as PostgreSQL, BigQuery, Athena, and many more.
 
 ## Links
 
- - [CloudQuery Quickstart Guide](https://www.cloudquery.io/docs/quickstart)
+ - [CloudQuery Quickstart Guide](https://cloudquery.io/docs/quickstart)
  - [Supported Tables](docs/tables/README.md)
 
 ## Configuration
 
-The following source configuration file will sync all data points for `mywebsite.com` to a PostgreSQL database. See [the CloudQuery Quickstart](https://www.cloudquery.io/docs/quickstart) for more information on how to configure the source and destination.
+The following source configuration file will sync all data points for `mywebsite.com` to a PostgreSQL database. See [CloudQuery Quickstart](https://cloudquery.io/docs/quickstart) for more information on how to configure the source and destination.
 
 ```yaml
 kind: source
 spec:
-  name: "simpleanalytics"
-  path: "simpleanalytics/simpleanalytics"
-  version: "${VERSION}"
-  # backend: "local" # use this to enable incremental syncing
+  name: "simple-analytics"
+  path: "simple-analytics/simple-analytics"
+  version: "v2.0.0"
+  # use this to enable incremental syncing
+  # backend_options:
+  #   table_name: "cq_state_simpleanalytics
+  #   connection: "@@plugins.DESTINATION_NAME.connection"
   tables: 
     ["*"]
   destinations: 
@@ -39,46 +42,49 @@ spec:
 
 ### Plugin Spec
 
-- `user_id` (string, required):
+- `user_id` (`string`) (required)
 
   A user ID from Simple Analytics, obtained from the [account settings](https://simpleanalytics.com/account) page. It should start with `sa_user_id...`
 
-- `api_key` (string, required):
+- `api_key` (`string`) (required)
 
   An API Key from Simple Analytics, obtained from the [account settings](https://simpleanalytics.com/account) page. It should start with `sa_api_key...`
 
-- `websites` (array, required):
+- `websites` (`list`) (required)
 
   A list of websites to sync data for. Each website should have the following fields:
 
-    - `hostname` (string, required):
+    - `hostname` (`string`) (required)
 
       The hostname of the website to sync data for. This should be the same as the hostname in Simple Analytics.
 
-    - `metadata_fields` (array[string], optional):
+    - `metadata_fields` (`[]string`) (optional)
 
       A list of metadata fields to sync, e.g. `["path_text", "created_at_time"]`. If not specified, no metadata fields will be synced.
 
-- `start_date` (string, optional):
+- `start_date` (`string`) (optional)
 
   The date to start syncing data from. If not specified, the plugin will sync data from the beginning of time (or use a start time defined by `period`, if set).
 
-- `end_date` (string, optional): 
+- `end_date` (`string`) (optional) 
 
   The date to stop syncing data at. If not specified, the plugin will sync data until the current date.
 
-- `period` (string, optional):
+- `period` (`string`) (optional)
 
   The duration of the time window to fetch historical data for, in days, months or years. It is used to calculate `start_date` if it is not specified. If `start_date` is specified, duration is ignored. Examples:
     - `7d`: last 7 days
     - `3m`: last 3 months
     - `1y`: last year
 
+- `concurrency` (`integer`) (optional`) (default: `1000`)
+
+  Best effort maximum number of Go routines to use. Lower this number to reduce memory usage.
 
 ## Incremental Syncing
 
 The Simple Analytics plugin supports incremental syncing. This means that only new data points will be fetched from Simple Analytics and loaded into your destination. This is done by keeping track of the last date a sync was done, and only fetching new data from that date onwards.
-To enable this, `backend` option must be set in the spec (as shown in the example config above). By default, incremental syncing is turned off. Also note that this will introduce duplicates, unless the destination is using `overwrite-delete-stale` mode. Care should be taken to remove these duplicates after loading them. For more information, see [Managing Incremental Tables](/docs/advanced-topics/managing-incremental-tables).
+To enable this, `backend_options` must be set in the spec (as shown in the example config above). By default, incremental syncing is turned off. Also note that this will introduce duplicates, unless the destination is using `overwrite-delete-stale` mode. Care should be taken to remove these duplicates after loading them. For more information, see [Managing Incremental Tables](https://cloudquery.io/docs/advanced-topics/managing-incremental-tables).
 
 ## Example Queries
 
@@ -172,7 +178,25 @@ make gen-docs
 ### Release a new version
 
 1. Run `git tag v1.0.0` to create a new tag for the release (replace `v1.0.0` with the new version number)
-2. Run `git push origin v1.0.0` to push the tag to GitHub  
+2. Run `git push origin v1.0.0` to push the tag to GitHub
 
 Once the tag is pushed, a new GitHub Actions workflow will be triggered to build the release binaries and create the new release on GitHub.
 To customize the release notes, see the Go releaser [changelog configuration docs](https://goreleaser.com/customization/changelog/#changelog).
+
+### Publish a new version to the Cloudquery Hub
+
+After tagging a release, you can build and publish a new version to the [Cloudquery Hub](https://hub.cloudquery.io/) by running the following commands.
+Replace `v1.0.0` with the new version number.
+
+```bash
+# -m parameter adds release notes message, output is created in dist/ directory
+go run main.go package -m "Release v1.0.0" v1.0.0 .
+
+# Login to cloudquery hub and publish the new version
+cloudquery login -t simple-analytics
+cloudquery plugin publish --finalize
+```
+
+After publishing the new version, it will [show up](https://hub.cloudquery.io/plugins/source/simple-analytics/simple-analytics) in the [hub](https://hub.cloudquery.io/).
+
+For more information please refer to the official [Publishing a Plugin to the Hub](https://cloudquery.io/docs/developers/publishing-a-plugin-to-the-hub) guide.

client/client.go

@@ -1,22 +1,17 @@
 package client
 
 import (
-	"context"
-	"fmt"
 	"strings"
 
-	"github.com/cloudquery/cq-source-simple-analytics/internal/simpleanalytics"
-	"github.com/cloudquery/plugin-sdk/backend"
-	"github.com/cloudquery/plugin-sdk/plugins/source"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/cloudquery/plugin-sdk/v4/state"
 	"github.com/rs/zerolog"
+	"github.com/simpleanalytics/cq-source-simple-analytics/internal/simpleanalytics"
 )
 
 type Client struct {
 	Logger   zerolog.Logger
 	SAClient *simpleanalytics.Client
-	Backend  backend.Backend
+	Backend  state.Client
 	Spec     Spec
 	Website  WebsiteSpec
 }
@@ -35,22 +30,11 @@ func (c *Client) withWebsite(website WebsiteSpec) *Client {
 	}
 }
 
-func New(_ context.Context, logger zerolog.Logger, s specs.Source, opts source.Options) (schema.ClientMeta, error) {
-	var pluginSpec Spec
-	if err := s.UnmarshalSpec(&pluginSpec); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal plugin spec: %w", err)
-	}
-	err := pluginSpec.Validate()
-	if err != nil {
-		return nil, fmt.Errorf("failed to validate plugin spec: %w", err)
-	}
-	pluginSpec.SetDefaults()
-
-	saClient := simpleanalytics.NewClient(pluginSpec.UserID, pluginSpec.APIKey)
+func New(logger zerolog.Logger, spec Spec, services *simpleanalytics.Client, bk state.Client) *Client {
 	return &Client{
 		Logger:   logger,
-		Backend:  opts.Backend,
-		Spec:     pluginSpec,
-		SAClient: saClient,
-	}, nil
+		SAClient: services,
+		Backend:  bk,
+		Spec:     spec,
+	}
 }

client/multiplexers.go

@@ -1,6 +1,6 @@
 package client
 
-import "github.com/cloudquery/plugin-sdk/schema"
+import "github.com/cloudquery/plugin-sdk/v4/schema"
 
 func WebsiteMultiplex(meta schema.ClientMeta) []schema.ClientMeta {
 	var l = make([]schema.ClientMeta, 0)

client/spec.go

@@ -41,6 +41,8 @@ type Spec struct {
 	// It is used to calculate start_time if it is not specified. If start_time is specified,
 	// duration is ignored.
 	PeriodStr string `json:"period"`
+
+	Concurrency int `json:"concurrency,omitempty"`
 }
 
 type WebsiteSpec struct {
@@ -91,6 +93,9 @@ func (s *Spec) SetDefaults() {
 	if s.EndDateStr == "" {
 		s.EndDateStr = time.Now().Format(AllowedTimeLayout)
 	}
+	if s.Concurrency < 1 {
+		s.Concurrency = 1000
+	}
 }
 
 func (s Spec) StartTime() time.Time {

client/testing.go

@@ -7,57 +7,48 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cloudquery/cq-source-simple-analytics/internal/simpleanalytics"
-	"github.com/cloudquery/plugin-sdk/plugins/source"
-	"github.com/cloudquery/plugin-sdk/schema"
-	"github.com/cloudquery/plugin-sdk/specs"
+	"github.com/cloudquery/plugin-sdk/v4/plugin"
+	"github.com/cloudquery/plugin-sdk/v4/scheduler"
+	"github.com/cloudquery/plugin-sdk/v4/schema"
+	"github.com/cloudquery/plugin-sdk/v4/transformers"
 	"github.com/rs/zerolog"
+	"github.com/simpleanalytics/cq-source-simple-analytics/internal/simpleanalytics"
 )
 
 func TestHelper(t *testing.T, table *schema.Table, ts *httptest.Server) {
-	version := "vDev"
 	table.IgnoreInTests = false
 	t.Helper()
+
 	l := zerolog.New(zerolog.NewTestWriter(t)).Output(
 		zerolog.ConsoleWriter{Out: os.Stderr, TimeFormat: time.StampMicro},
 	).Level(zerolog.DebugLevel).With().Timestamp().Logger()
-	newTestExecutionClient := func(ctx context.Context, logger zerolog.Logger, spec specs.Source, opts source.Options) (schema.ClientMeta, error) {
-		saClient := simpleanalytics.NewClient("test", "test", simpleanalytics.WithBaseURL(ts.URL), simpleanalytics.WithHTTPClient(ts.Client()))
-		s := Spec{
-			UserID: "test",
-			APIKey: "test",
-			Websites: []WebsiteSpec{
-				{
-					Hostname:       "test.com",
-					MetadataFields: []string{"metadata_text", "metadata_int"},
-				},
+	sched := scheduler.NewScheduler(scheduler.WithLogger(l))
+
+	spec := &Spec{
+		UserID: "test",
+		APIKey: "test",
+		Websites: []WebsiteSpec{
+			{
+				Hostname:       "test.com",
+				MetadataFields: []string{"metadata_text", "metadata_int"},
 			},
-		}
-		s.SetDefaults()
-		err := s.Validate()
-		if err != nil {
-			return nil, err
-		}
-		return &Client{
-			Logger:   l,
-			SAClient: saClient,
-			Backend:  opts.Backend,
-			Spec:     s,
-		}, nil
-	}
-	p := source.NewPlugin(
-		table.Name,
-		version,
-		[]*schema.Table{
-			table,
 		},
-		newTestExecutionClient)
-	p.SetLogger(l)
-	source.TestPluginSync(t, p, specs.Source{
-		Name:         "dev",
-		Path:         "cloudquery/dev",
-		Version:      version,
-		Tables:       []string{table.Name},
-		Destinations: []string{"mock-destination"},
-	})
+	}
+	spec.SetDefaults()
+	if err := spec.Validate(); err != nil {
+		t.Fatalf("failed to validate spec: %v", err)
+	}
+
+	saClient := simpleanalytics.NewClient(spec.UserID, spec.APIKey, simpleanalytics.WithBaseURL(ts.URL), simpleanalytics.WithHTTPClient(ts.Client()))
+	c := New(l, *spec, saClient, nil)
+
+	tables := schema.Tables{table}
+	if err := transformers.TransformTables(tables); err != nil {
+		t.Fatal(err)
+	}
+	messages, err := sched.SyncAll(context.Background(), c, tables)
+	if err != nil {
+		t.Fatalf("failed to sync: %v", err)
+	}
+	plugin.ValidateNoEmptyColumns(t, tables, messages)
 }