docs(samples): add product import samples (#149)

* feat: product_number variable is replaced with product_id
Import product code samples are added

* revert changes in README

* fix typo in TEST_RESOURCES_SETUP_CLEANUP.md

* include project number in import_products_big_query_table.py

* use project_number in import_products_gcs.py; use f-string

* use project_number in import_products_inline_source.py

* use project_number in product/setup/setup_cleanup.py

* use project_number in search_simple_query.py

* use project_number in search_with_boost_spec.py

* use project_number in search_with_filtering.py

* use project number in search_with_ordering.py

* use project_number in search_with_pagination.py

* use project_number in search_with_query_expansion_spec.py

* use project_number in test_resources_recovery

* use project_number in search_with_facet_spec.py

* fix typo in f-string

* fix typos

* use correct path to resources

* lint

* revert change to paths

* resolve error where bq table doesn't exist

* make setup more robust

* lint

* import product from inline source is fixed

* use google.cloud.bigquery client

* update test to reflect changes

* lint

Co-authored-by: Karl Weinmeister <[email protected]>
Co-authored-by: Anthonios Partheniou <[email protected]>

Author: 3 people

generated_samples/interactive-tutorials/README.md

@@ -72,19 +72,19 @@ To run a code sample from the Cloud Shell, you need to be authenticated using th
     export GOOGLE_APPLICATION_CREDENTIALS=~/key.json
     ```
 
-### Set the PROJECT_NUMBER and PROJECT_ID environment variables
+### Set the GOOGLE_CLOUD_PROJECT_NUMBER and GOOGLE_CLOUD_PROJECT environment variables
 
 You will run the code samples in your own Google Cloud project. To use the **project_number** and **project_id** in every request to the Retail API, you should first specify them as environment variables.
 
 1. Find the project number and project ID in the Project Info card displayed on **Home/Dashboard**.
 
 1. Set the **project_number** with the following command:
     ```bash
-    export PROJECT_NUMBER=<YOUR_PROJECT_NUMBER>
+    export GOOGLE_CLOUD_PROJECT_NUMBER=<YOUR_PROJECT_NUMBER>
     ```
 1. Set the **project_id** with the following command:
     ```bash
-    export PROJECT_ID=<YOUR_PROJECT_ID>
+    export GOOGLE_CLOUD_PROJECT=<YOUR_PROJECT_ID>
     ```
 
 ### Install Google Cloud Retail libraries

generated_samples/interactive-tutorials/TEST_RESOURCES_SETUP_CLEANUP.md

@@ -4,7 +4,7 @@
 
 To successfully import the catalog data for tests, the following environment variables should be set:
  - GOOGLE_CLOUD_PROJECT_NUMBER
- - GOOGLE_CLOUD_PROJECT_ID 
+ - GOOGLE_CLOUD_PROJECT 
  - BUCKET_NAME
  - EVENTS_BUCKET_NAME
 These values are stored in the Secret Manager and will be submitted as 

generated_samples/interactive-tutorials/product/import_products_big_query_table.py

@@ -0,0 +1,101 @@
+# Copyright 2021 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START retail_import_products_from_big_query]
+# Import products into a catalog from big query table using Retail API
+#
+import os
+import time
+
+from google.cloud.retail import (
+    BigQuerySource,
+    ImportProductsRequest,
+    ProductInputConfig,
+    ProductServiceClient,
+)
+
+project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"]
+project_id = os.environ["GOOGLE_CLOUD_PROJECT"]
+
+default_catalog = f"projects/{project_number}/locations/global/catalogs/default_catalog/branches/default_branch"
+dataset_id = "products"
+table_id = "products"
+
+
+# TO CHECK ERROR HANDLING USE THE TABLE WITH INVALID PRODUCTS:
+# table_id = "products_some_invalid"
+
+
+# get import products from big query request
+def get_import_products_big_query_request(reconciliation_mode):
+    # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE:
+    # default_catalog = "invalid_catalog_name"
+    big_query_source = BigQuerySource()
+    big_query_source.project_id = project_id
+    big_query_source.dataset_id = dataset_id
+    big_query_source.table_id = table_id
+    big_query_source.data_schema = "product"
+
+    input_config = ProductInputConfig()
+    input_config.big_query_source = big_query_source
+
+    import_request = ImportProductsRequest()
+    import_request.parent = default_catalog
+    import_request.reconciliation_mode = reconciliation_mode
+    import_request.input_config = input_config
+
+    print("---import products from big query table request---")
+    print(import_request)
+
+    return import_request
+
+
+# call the Retail API to import products
+def import_products_from_big_query():
+    # TRY THE FULL RECONCILIATION MODE HERE:
+    reconciliation_mode = ImportProductsRequest.ReconciliationMode.INCREMENTAL
+
+    import_big_query_request = get_import_products_big_query_request(
+        reconciliation_mode
+    )
+    big_query_operation = ProductServiceClient().import_products(
+        import_big_query_request
+    )
+
+    print("---the operation was started:----")
+    print(big_query_operation.operation.name)
+
+    while not big_query_operation.done():
+        print("---please wait till operation is done---")
+        time.sleep(30)
+    print("---import products operation is done---")
+
+    if big_query_operation.metadata is not None:
+        print("---number of successfully imported products---")
+        print(big_query_operation.metadata.success_count)
+        print("---number of failures during the importing---")
+        print(big_query_operation.metadata.failure_count)
+    else:
+        print("---operation.metadata is empty---")
+
+    if big_query_operation.result is not None:
+        print("---operation result:---")
+        print(big_query_operation.result())
+    else:
+        print("---operation.result is empty---")
+
+
+import_products_from_big_query()
+
+# [END retail_import_products_from_big_query]

generated_samples/interactive-tutorials/product/import_products_bq_test.py

@@ -0,0 +1,31 @@
+# Copyright 2021 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import subprocess
+
+
+def test_import_products_bq():
+    output = str(
+        subprocess.check_output("python import_products_big_query_table.py", shell=True)
+    )
+
+    assert re.match(".*import products from big query table request.*", output)
+    assert re.match(".*the operation was started.*", output)
+    assert re.match(
+        ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*",
+        output,
+    )
+
+    assert re.match(".*number of successfully imported products.*316.*", output)

generated_samples/interactive-tutorials/product/import_products_gcs.py

@@ -0,0 +1,111 @@
+# Copyright 2021 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# [START retail_import_products_from_gcs]
+# Import products into a catalog from gcs using Retail API
+#
+import os
+import time
+
+from google.cloud.retail import (
+    GcsSource,
+    ImportErrorsConfig,
+    ImportProductsRequest,
+    ProductInputConfig,
+    ProductServiceClient,
+)
+
+
+# Read the project number from the environment variable
+project_number = os.environ["GOOGLE_CLOUD_PROJECT_NUMBER"]
+project_id = os.environ["GOOGLE_CLOUD_PROJECT"]
+bucket_name = os.environ["BUCKET_NAME"]
+
+# You can change the branch here. The "default_branch" is set to point to the branch "0"
+default_catalog = f"projects/{project_number}/locations/global/catalogs/default_catalog/branches/default_branch"
+
+gcs_bucket = f"gs://{bucket_name}"
+gcs_errors_bucket = f"{gcs_bucket}/error"
+gcs_products_object = "products.json"
+
+
+# TO CHECK ERROR HANDLING USE THE JSON WITH INVALID PRODUCT
+# gcs_products_object = "products_some_invalid.json"
+
+
+# get import products from gcs request
+def get_import_products_gcs_request(gcs_object_name: str):
+    # TO CHECK ERROR HANDLING PASTE THE INVALID CATALOG NAME HERE:
+    # default_catalog = "invalid_catalog_name"
+    gcs_source = GcsSource()
+    gcs_source.input_uris = [f"{gcs_bucket}/{gcs_object_name}"]
+
+    input_config = ProductInputConfig()
+    input_config.gcs_source = gcs_source
+    print("GRS source:")
+    print(gcs_source.input_uris)
+
+    errors_config = ImportErrorsConfig()
+    errors_config.gcs_prefix = gcs_errors_bucket
+
+    import_request = ImportProductsRequest()
+    import_request.parent = default_catalog
+    import_request.reconciliation_mode = (
+        ImportProductsRequest.ReconciliationMode.INCREMENTAL
+    )
+    import_request.input_config = input_config
+    import_request.errors_config = errors_config
+
+    print("---import products from google cloud source request---")
+    print(import_request)
+
+    return import_request
+
+
+# call the Retail API to import products
+def import_products_from_gcs():
+    import_gcs_request = get_import_products_gcs_request(gcs_products_object)
+    gcs_operation = ProductServiceClient().import_products(import_gcs_request)
+
+    print("---the operation was started:----")
+    print(gcs_operation.operation.name)
+
+    while not gcs_operation.done():
+        print("---please wait till operation is done---")
+        time.sleep(30)
+    print("---import products operation is done---")
+
+    if gcs_operation.metadata is not None:
+        print("---number of successfully imported products---")
+        print(gcs_operation.metadata.success_count)
+        print("---number of failures during the importing---")
+        print(gcs_operation.metadata.failure_count)
+    else:
+        print("---operation.metadata is empty---")
+
+    if gcs_operation.result is not None:
+        print("---operation result:---")
+        print(gcs_operation.result())
+    else:
+        print("---operation.result is empty---")
+
+    # The imported products needs to be indexed in the catalog before they become available for search.
+    print(
+        "Wait 2-5 minutes till products become indexed in the catalog, after that they will be available for search"
+    )
+
+
+import_products_from_gcs()
+
+# [END retail_import_products_from_gcs]

generated_samples/interactive-tutorials/product/import_products_gcs_test.py

@@ -0,0 +1,30 @@
+# Copyright 2021 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import subprocess
+
+
+def test_import_products_gcs():
+    output = str(subprocess.check_output("python import_products_gcs.py", shell=True))
+
+    assert re.match(".*import products from google cloud source request.*", output)
+    assert re.match('.*input_uris: "gs://.*/products.json".*', output)
+    assert re.match(".*the operation was started.*", output)
+    assert re.match(
+        ".*projects/.*/locations/global/catalogs/default_catalog/branches/0/operations/import-products.*",
+        output,
+    )
+
+    assert re.match(".*number of successfully imported products.*316.*", output)