From 967863e0a3755dd30478467c18ed71f64cef8e72 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 2 Oct 2018 17:40:35 +0300 Subject: [PATCH 01/48] Remote package upload API implementation Any user with permission to enter admin panel can upload packages to the deployment server. This will be used by a (not yet implemented) tool in qt application manager for automatic package upload from qtcreator. Fixes: AUTOSUITE-623 Change-Id: I5aba9d16480e2161e5e633359070004f66f2b897 Reviewed-by: Dominik Holland --- appstore/urls.py | 1 + store/api.py | 60 +++++++- store/authdecorators.py | 135 ++++++++++++++++++ .../commands/store-upload-package.py | 39 +---- store/models.py | 38 ++++- 5 files changed, 233 insertions(+), 40 deletions(-) create mode 100644 store/authdecorators.py diff --git a/appstore/urls.py b/appstore/urls.py index 081acf9..327c28b 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -46,6 +46,7 @@ url(r'^app/download/(.*)$', 'store.api.appDownload'), url(r'^category/list$', 'store.api.categoryList'), url(r'^category/icon$', 'store.api.categoryIcon'), + url(r'^upload$', 'store.api.upload'), ) diff --git a/store/api.py b/store/api.py index 9734f6e..f2d911e 100644 --- a/store/api.py +++ b/store/api.py @@ -30,19 +30,19 @@ ############################################################################# import os -import tempfile -import datetime import shutil -import json from django.conf import settings from django.db.models import Q, Count from django.http import HttpResponse, HttpResponseForbidden, Http404, JsonResponse from django.contrib import auth from django.template import Context, loader +from django.views.decorators.csrf import csrf_exempt +from authdecorators import logged_in_or_basicauth, is_staff_member -from models import App, Category, Vendor -from utilities import parsePackageMetadata, packagePath, iconPath, downloadPath, addSignatureToPackage, validateTag +from models import App, Category, Vendor, savePackageFile +from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage +from utilities import packagePath, iconPath, downloadPath, validateTag def hello(request): @@ -105,6 +105,56 @@ def logout(request): return JsonResponse({'status': status}) +@csrf_exempt +@logged_in_or_basicauth() +@is_staff_member() +def upload(request): + status = 'ok' + try: + try: + description = request.REQUEST["description"] + except: + raise Exception('no description') + try: + shortdescription = request.REQUEST["short-description"] + except: + raise Exception('no short description') + try: + category_name = request.REQUEST["category"] + except: + raise Exception('no category') + try: + vendor_name = request.REQUEST["vendor"] + except: + raise Exception('no vendor') + + if request.method == 'POST' and request.FILES['package']: + myfile = request.FILES['package'] + category = Category.objects.all().filter(name__exact=category_name) + vendor = Vendor.objects.all().filter(name__exact=vendor_name) + if len(category) == 0: + raise Exception('Non-existing category') + if len(vendor) == 0: + raise Exception('Non-existing vendor') + + try: + pkgdata = parseAndValidatePackageMetadata(myfile) + except: + raise Exception('Package validation failed') + + myfile.seek(0) + try: + savePackageFile(pkgdata, myfile, category[0], vendor[0], description, shortdescription) + except Exception as error: + raise Exception(error) + else: + raise Exception('no package to upload') + + except Exception as error: + status = str(error) + return JsonResponse({'status': status}) + + def appList(request): apps = App.objects.all() if 'filter' in request.REQUEST: diff --git a/store/authdecorators.py b/store/authdecorators.py new file mode 100644 index 0000000..fbbc01d --- /dev/null +++ b/store/authdecorators.py @@ -0,0 +1,135 @@ +############################################################################# +## +## Copyright (C) 2018 Luxoft +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the Neptune Deployment Server +## +## $QT_BEGIN_LICENSE:GPL-QTAS$ +## Commercial License Usage +## Licensees holding valid commercial Qt Automotive Suite licenses may use +## this file in accordance with the commercial license agreement provided +## with the Software or, alternatively, in accordance with the terms +## contained in a written agreement between you and The Qt Company. For +## licensing terms and conditions see https://www.qt.io/terms-conditions. +## For further information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 or (at your option) any later version +## approved by the KDE Free Qt Foundation. The licenses are as published by +## the Free Software Foundation and appearing in the file LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +## SPDX-License-Identifier: GPL-3.0 +## +############################################################################# + +# Code taken from: https://www.djangosnippets.org/snippets/243/ +# Reuse and licensing is permitted by TOS: https://www.djangosnippets.org/about/tos/ + +import base64 + +from django.http import HttpResponse +from django.contrib.auth import authenticate, login + +############################################################################# + +def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): + """ + This is a helper function used by both 'logged_in_or_basicauth' and + 'has_perm_or_basicauth' that does the service of determining if they + are already logged in or if they have provided proper http-authorization + and returning the view if all goes well, otherwise responding with a 401. + """ + if test_func(request.user): + # Already logged in, just return the view. + # + return view(request, *args, **kwargs) + + # They are not logged in. See if they provided login credentials + # + if 'HTTP_AUTHORIZATION' in request.META: + auth = request.META['HTTP_AUTHORIZATION'].split() + if len(auth) == 2: + # NOTE: We are only support basic authentication for now. + # + if auth[0].lower() == "basic": + uname, passwd = base64.b64decode(auth[1]).split(':') + user = authenticate(username=uname, password=passwd) + if user is not None: + if user.is_active: + login(request, user) + request.user = user + if test_func(request.user): + return view(request, *args, **kwargs) + + # Either they did not provide an authorization header or + # something in the authorization attempt failed. Send a 401 + # back to them to ask them to authenticate. + # + response = HttpResponse() + response.status_code = 401 + response['WWW-Authenticate'] = 'Basic realm="%s"' % realm + return response + + +############################################################################# + +def logged_in_or_basicauth(realm=""): + """ + A simple decorator that requires a user to be logged in and in staff group. + If they are not logged in the request is examined for a 'authorization' header. + + If the header is present it is tested for basic authentication and + the user is logged in with the provided credentials. + + If the header is not present a http 401 is sent back to the + requester to provide credentials. + + The purpose of this is that in several django projects I have needed + several specific views that need to support basic authentication, yet the + web site as a whole used django's provided authentication. + + The uses for this are for urls that are access programmatically such as + by rss feed readers, yet the view requires a user to be logged in. Many rss + readers support supplying the authentication credentials via http basic + auth (and they do NOT support a redirect to a form where they post a + username/password.) + + Use is simple: + + @logged_in_or_basicauth + def your_view: + ... + + You can provide the name of the realm to ask for authentication within. + """ + + def view_decorator(func): + def wrapper(request, *args, **kwargs): + return view_or_basicauth(func, request, + lambda u: u.is_authenticated(), + realm, *args, **kwargs) + + return wrapper + + return view_decorator + +def is_staff_member(): + def view_decorator(func): + def wrapper(request, *args, **kwargs): + if request.user.is_staff: + return func(request, *args, **kwargs) + else: + response = HttpResponse() + response.status_code = 403 + return response + + return wrapper + return view_decorator + diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 1468d27..81f96fa 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -33,8 +33,8 @@ from django.core.management.base import BaseCommand, CommandError from django.core.files.base import ContentFile -from store.models import App, Category, Vendor -from store.utilities import parseAndValidatePackageMetadata, packagePath, makeTagList, writeTempIcon +from store.models import App, Category, Vendor, savePackageFile +from store.utilities import parseAndValidatePackageMetadata from optparse import make_option @@ -83,38 +83,9 @@ def handle(self, *args, **options): return 0 packagefile.seek(0) - appId = pkgdata['info']['id'] - name = pkgdata['storeName'] - architecture = pkgdata['architecture'] description = options['description'] - tags = makeTagList(pkgdata) - - success, error = writeTempIcon(appId, architecture, pkgdata['icon']) - if not success: - raise CommandError(error) - - exists = False - app = None try: - app = App.objects.get(appid__exact=appId, architecture__exact= architecture) - exists = True - except App.DoesNotExist: - pass + savePackageFile(pkgdata, ContentFile(packagefile.read()), category[0], vendor[0], description, description) + except Exception as error: + raise CommandError(error) - if exists: - app.appid = appId - app.category = category[0] - app.vendor = vendor[0] - app.name = name - app.tags = tags - app.description = app.briefDescription = description - app.architecture = architecture - app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read())) - app.save() - else: - app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor[0], - category=category[0], appid=appId, - briefDescription=description, description=description, - architecture=architecture) - app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read())) - app.save() diff --git a/store/models.py b/store/models.py index 802d141..63b1d89 100644 --- a/store/models.py +++ b/store/models.py @@ -36,7 +36,7 @@ from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage -from utilities import packagePath +from utilities import packagePath, writeTempIcon, makeTagList class Category(models.Model): @@ -142,3 +142,39 @@ def save(self, *args, **kwargs): pass super(App, self).save(*args, **kwargs) + +def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescription): + appId = pkgdata['info']['id'] + name = pkgdata['storeName'] + architecture = pkgdata['architecture'] + tags = makeTagList(pkgdata) + success, error = writeTempIcon(appId, architecture, pkgdata['icon']) + if not success: + raise Exception(error) + + exists = False + app = None + try: + app = App.objects.get(appid__exact=appId, architecture__exact=architecture) + exists = True + except App.DoesNotExist: + pass + + if exists: + app.appid = appId + app.category = category + app.vendor = vendor + app.name = name + app.tags = tags + app.description = description + app.briefDescription = shortdescription + app.architecture = architecture + app.file.save(packagePath(appId, architecture), pkgfile) + app.save() + else: + app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, + category=category, appid=appId, + briefDescription=shortdescription, description=description, + architecture=architecture) + app.file.save(packagePath(appId, architecture), pkgfile) + app.save() From b0dd327dfa5b3cb0aa3157119f306bfab5515edc Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 24 Oct 2018 20:55:11 +0300 Subject: [PATCH 02/48] Update for architecture and bitness detection and processing This is done to match QSysInfo::currentCpuArchitecture(), kernelVersion() and buildAbi() See implementation in neptune3-ui for details. Without this code, neptune3-ui would be unable to get native applications. Change-Id: Ic336b364575bc22c1812693fea254d4898f1ee6f Fixes: AUTOSUITE-670 Reviewed-by: Dominik Holland --- store/api.py | 3 ++- store/osandarch.py | 51 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/store/api.py b/store/api.py index f2d911e..6e951a4 100644 --- a/store/api.py +++ b/store/api.py @@ -43,6 +43,7 @@ from models import App, Category, Vendor, savePackageFile from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage from utilities import packagePath, iconPath, downloadPath, validateTag +from osandarch import normalizeArch def hello(request): @@ -64,7 +65,7 @@ def hello(request): break request.session[j] = taglist if 'architecture' in request.REQUEST: - request.session['architecture'] = request.REQUEST['architecture'] + request.session['architecture'] = normalizeArch(request.REQUEST['architecture']) else: request.session['architecture'] = '' return JsonResponse({'status': status}) diff --git a/store/osandarch.py b/store/osandarch.py index f3aa388..c7d15a6 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -42,6 +42,8 @@ # PE32+ executable (DLL) (GUI) x86-64, for MS Windows # PE32+ executable (GUI) x86-64, for MS Windows +import re + def parseMachO(str): # os, arch, bits, endianness if " universal " in str: # Universal binary - not supported @@ -50,7 +52,7 @@ def parseMachO(str): # os, arch, bits, endianness arch = str.split(' ') arch = arch[2] bits = str.split(' ')[1].replace('-bit', '') - endianness = "lsb" + endianness = "little_endian" return [os, arch, bits, endianness] @@ -65,24 +67,28 @@ def parsePE32(str): bits = '64' if arch == '80386': arch = 'i386' - endianness = "lsb" + endianness = "little_endian" return [os, arch, bits, endianness] -def parseElfArch(str, architecture): +def parseElfArch(str, architecture, bits): architecture = architecture.strip() if architecture.startswith("ARM"): if 'aarch64' in architecture: - return 'aarch64' + return 'arm64' if 'armhf' in str: # this does not work for some reason - from_file() returns longer data than from_buffer() - needs fix - return 'armhf' + return 'arm' # because qt does not report it directly elif architecture.startswith("Intel"): if '80386' in architecture: return 'i386' elif architecture.startswith("IBM S/390"): return 's/390' elif "PowerPC" in architecture: - return 'powerpc' + if bits == "64": + return "power64" + else: + return 'power' + # sparc architecture is currently ignored (should be handled similar to power?) return architecture.lower() @@ -90,9 +96,15 @@ def parseElf(str): os = "Linux" arch = str.split(',') arch = arch[1] - arch = parseElfArch(str, arch) bits = str.split(' ')[1].replace('-bit', '') - endianness = str.split(' ')[2].lower() + arch = parseElfArch(str, arch, bits) + endian = str.split(' ')[2].lower() + if endian == "msb": + endianness = "big_endian" + elif endian == "lsb": + endianness = "little_endian" + else: + raise Exception("Unrecognised endianness") return [os, arch, bits, endianness] @@ -117,3 +129,26 @@ def getOsArch(str): if os: return result return None + +def normalizeArch(inputArch): + """ + This function brings requested architecture to common form (currently just parses the bits part and turns it into 32/64) + Input string format is: arch-endianness-word_size-kernelType + + """ + parts = inputArch.split('-') + #Drop anything non-numeric from word_size field + parts[2]=re.sub(r"\D", "", parts[2]) + #Transform kernelType into binary format + temp = parts[3] + if "win" in temp: + parts[3]="pe32" + elif "linux" in temp: + parts[3]="elf" + elif "freebsd" in temp: #How do we treat QNX? + parts[3]="elf" + elif "darwin" in temp: + parts[3]="mach_o" + #Rejoin new architecture + arch = '-'.join(parts) + return arch From bf62f449e41073fee09f87df858489fab23cf018 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 2 Oct 2018 17:40:35 +0300 Subject: [PATCH 03/48] Remote package upload API implementation Any user with permission to enter admin panel can upload packages to the deployment server. This will be used by a (not yet implemented) tool in qt application manager for automatic package upload from qtcreator. Fixes: AUTOSUITE-623 Change-Id: I5aba9d16480e2161e5e633359070004f66f2b897 Reviewed-by: Dominik Holland --- appstore/urls.py | 1 + store/api.py | 60 +++++++- store/authdecorators.py | 135 ++++++++++++++++++ .../commands/store-upload-package.py | 39 +---- store/models.py | 38 ++++- 5 files changed, 233 insertions(+), 40 deletions(-) create mode 100644 store/authdecorators.py diff --git a/appstore/urls.py b/appstore/urls.py index 081acf9..327c28b 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -46,6 +46,7 @@ url(r'^app/download/(.*)$', 'store.api.appDownload'), url(r'^category/list$', 'store.api.categoryList'), url(r'^category/icon$', 'store.api.categoryIcon'), + url(r'^upload$', 'store.api.upload'), ) diff --git a/store/api.py b/store/api.py index 9734f6e..f2d911e 100644 --- a/store/api.py +++ b/store/api.py @@ -30,19 +30,19 @@ ############################################################################# import os -import tempfile -import datetime import shutil -import json from django.conf import settings from django.db.models import Q, Count from django.http import HttpResponse, HttpResponseForbidden, Http404, JsonResponse from django.contrib import auth from django.template import Context, loader +from django.views.decorators.csrf import csrf_exempt +from authdecorators import logged_in_or_basicauth, is_staff_member -from models import App, Category, Vendor -from utilities import parsePackageMetadata, packagePath, iconPath, downloadPath, addSignatureToPackage, validateTag +from models import App, Category, Vendor, savePackageFile +from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage +from utilities import packagePath, iconPath, downloadPath, validateTag def hello(request): @@ -105,6 +105,56 @@ def logout(request): return JsonResponse({'status': status}) +@csrf_exempt +@logged_in_or_basicauth() +@is_staff_member() +def upload(request): + status = 'ok' + try: + try: + description = request.REQUEST["description"] + except: + raise Exception('no description') + try: + shortdescription = request.REQUEST["short-description"] + except: + raise Exception('no short description') + try: + category_name = request.REQUEST["category"] + except: + raise Exception('no category') + try: + vendor_name = request.REQUEST["vendor"] + except: + raise Exception('no vendor') + + if request.method == 'POST' and request.FILES['package']: + myfile = request.FILES['package'] + category = Category.objects.all().filter(name__exact=category_name) + vendor = Vendor.objects.all().filter(name__exact=vendor_name) + if len(category) == 0: + raise Exception('Non-existing category') + if len(vendor) == 0: + raise Exception('Non-existing vendor') + + try: + pkgdata = parseAndValidatePackageMetadata(myfile) + except: + raise Exception('Package validation failed') + + myfile.seek(0) + try: + savePackageFile(pkgdata, myfile, category[0], vendor[0], description, shortdescription) + except Exception as error: + raise Exception(error) + else: + raise Exception('no package to upload') + + except Exception as error: + status = str(error) + return JsonResponse({'status': status}) + + def appList(request): apps = App.objects.all() if 'filter' in request.REQUEST: diff --git a/store/authdecorators.py b/store/authdecorators.py new file mode 100644 index 0000000..fbbc01d --- /dev/null +++ b/store/authdecorators.py @@ -0,0 +1,135 @@ +############################################################################# +## +## Copyright (C) 2018 Luxoft +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the Neptune Deployment Server +## +## $QT_BEGIN_LICENSE:GPL-QTAS$ +## Commercial License Usage +## Licensees holding valid commercial Qt Automotive Suite licenses may use +## this file in accordance with the commercial license agreement provided +## with the Software or, alternatively, in accordance with the terms +## contained in a written agreement between you and The Qt Company. For +## licensing terms and conditions see https://www.qt.io/terms-conditions. +## For further information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 or (at your option) any later version +## approved by the KDE Free Qt Foundation. The licenses are as published by +## the Free Software Foundation and appearing in the file LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +## SPDX-License-Identifier: GPL-3.0 +## +############################################################################# + +# Code taken from: https://www.djangosnippets.org/snippets/243/ +# Reuse and licensing is permitted by TOS: https://www.djangosnippets.org/about/tos/ + +import base64 + +from django.http import HttpResponse +from django.contrib.auth import authenticate, login + +############################################################################# + +def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): + """ + This is a helper function used by both 'logged_in_or_basicauth' and + 'has_perm_or_basicauth' that does the service of determining if they + are already logged in or if they have provided proper http-authorization + and returning the view if all goes well, otherwise responding with a 401. + """ + if test_func(request.user): + # Already logged in, just return the view. + # + return view(request, *args, **kwargs) + + # They are not logged in. See if they provided login credentials + # + if 'HTTP_AUTHORIZATION' in request.META: + auth = request.META['HTTP_AUTHORIZATION'].split() + if len(auth) == 2: + # NOTE: We are only support basic authentication for now. + # + if auth[0].lower() == "basic": + uname, passwd = base64.b64decode(auth[1]).split(':') + user = authenticate(username=uname, password=passwd) + if user is not None: + if user.is_active: + login(request, user) + request.user = user + if test_func(request.user): + return view(request, *args, **kwargs) + + # Either they did not provide an authorization header or + # something in the authorization attempt failed. Send a 401 + # back to them to ask them to authenticate. + # + response = HttpResponse() + response.status_code = 401 + response['WWW-Authenticate'] = 'Basic realm="%s"' % realm + return response + + +############################################################################# + +def logged_in_or_basicauth(realm=""): + """ + A simple decorator that requires a user to be logged in and in staff group. + If they are not logged in the request is examined for a 'authorization' header. + + If the header is present it is tested for basic authentication and + the user is logged in with the provided credentials. + + If the header is not present a http 401 is sent back to the + requester to provide credentials. + + The purpose of this is that in several django projects I have needed + several specific views that need to support basic authentication, yet the + web site as a whole used django's provided authentication. + + The uses for this are for urls that are access programmatically such as + by rss feed readers, yet the view requires a user to be logged in. Many rss + readers support supplying the authentication credentials via http basic + auth (and they do NOT support a redirect to a form where they post a + username/password.) + + Use is simple: + + @logged_in_or_basicauth + def your_view: + ... + + You can provide the name of the realm to ask for authentication within. + """ + + def view_decorator(func): + def wrapper(request, *args, **kwargs): + return view_or_basicauth(func, request, + lambda u: u.is_authenticated(), + realm, *args, **kwargs) + + return wrapper + + return view_decorator + +def is_staff_member(): + def view_decorator(func): + def wrapper(request, *args, **kwargs): + if request.user.is_staff: + return func(request, *args, **kwargs) + else: + response = HttpResponse() + response.status_code = 403 + return response + + return wrapper + return view_decorator + diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 1468d27..81f96fa 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -33,8 +33,8 @@ from django.core.management.base import BaseCommand, CommandError from django.core.files.base import ContentFile -from store.models import App, Category, Vendor -from store.utilities import parseAndValidatePackageMetadata, packagePath, makeTagList, writeTempIcon +from store.models import App, Category, Vendor, savePackageFile +from store.utilities import parseAndValidatePackageMetadata from optparse import make_option @@ -83,38 +83,9 @@ def handle(self, *args, **options): return 0 packagefile.seek(0) - appId = pkgdata['info']['id'] - name = pkgdata['storeName'] - architecture = pkgdata['architecture'] description = options['description'] - tags = makeTagList(pkgdata) - - success, error = writeTempIcon(appId, architecture, pkgdata['icon']) - if not success: - raise CommandError(error) - - exists = False - app = None try: - app = App.objects.get(appid__exact=appId, architecture__exact= architecture) - exists = True - except App.DoesNotExist: - pass + savePackageFile(pkgdata, ContentFile(packagefile.read()), category[0], vendor[0], description, description) + except Exception as error: + raise CommandError(error) - if exists: - app.appid = appId - app.category = category[0] - app.vendor = vendor[0] - app.name = name - app.tags = tags - app.description = app.briefDescription = description - app.architecture = architecture - app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read())) - app.save() - else: - app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor[0], - category=category[0], appid=appId, - briefDescription=description, description=description, - architecture=architecture) - app.file.save(packagePath(appId, architecture), ContentFile(packagefile.read())) - app.save() diff --git a/store/models.py b/store/models.py index 802d141..63b1d89 100644 --- a/store/models.py +++ b/store/models.py @@ -36,7 +36,7 @@ from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage -from utilities import packagePath +from utilities import packagePath, writeTempIcon, makeTagList class Category(models.Model): @@ -142,3 +142,39 @@ def save(self, *args, **kwargs): pass super(App, self).save(*args, **kwargs) + +def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescription): + appId = pkgdata['info']['id'] + name = pkgdata['storeName'] + architecture = pkgdata['architecture'] + tags = makeTagList(pkgdata) + success, error = writeTempIcon(appId, architecture, pkgdata['icon']) + if not success: + raise Exception(error) + + exists = False + app = None + try: + app = App.objects.get(appid__exact=appId, architecture__exact=architecture) + exists = True + except App.DoesNotExist: + pass + + if exists: + app.appid = appId + app.category = category + app.vendor = vendor + app.name = name + app.tags = tags + app.description = description + app.briefDescription = shortdescription + app.architecture = architecture + app.file.save(packagePath(appId, architecture), pkgfile) + app.save() + else: + app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, + category=category, appid=appId, + briefDescription=shortdescription, description=description, + architecture=architecture) + app.file.save(packagePath(appId, architecture), pkgfile) + app.save() From 97403e148e15e21b1f8d82e02df1c8ab7528025f Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 24 Oct 2018 20:55:11 +0300 Subject: [PATCH 04/48] Update for architecture and bitness detection and processing This is done to match QSysInfo::currentCpuArchitecture(), kernelVersion() and buildAbi() See implementation in neptune3-ui for details. Without this code, neptune3-ui would be unable to get native applications. Change-Id: Ic336b364575bc22c1812693fea254d4898f1ee6f Fixes: AUTOSUITE-670 Reviewed-by: Dominik Holland --- store/api.py | 3 ++- store/osandarch.py | 51 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/store/api.py b/store/api.py index f2d911e..6e951a4 100644 --- a/store/api.py +++ b/store/api.py @@ -43,6 +43,7 @@ from models import App, Category, Vendor, savePackageFile from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage from utilities import packagePath, iconPath, downloadPath, validateTag +from osandarch import normalizeArch def hello(request): @@ -64,7 +65,7 @@ def hello(request): break request.session[j] = taglist if 'architecture' in request.REQUEST: - request.session['architecture'] = request.REQUEST['architecture'] + request.session['architecture'] = normalizeArch(request.REQUEST['architecture']) else: request.session['architecture'] = '' return JsonResponse({'status': status}) diff --git a/store/osandarch.py b/store/osandarch.py index f3aa388..c7d15a6 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -42,6 +42,8 @@ # PE32+ executable (DLL) (GUI) x86-64, for MS Windows # PE32+ executable (GUI) x86-64, for MS Windows +import re + def parseMachO(str): # os, arch, bits, endianness if " universal " in str: # Universal binary - not supported @@ -50,7 +52,7 @@ def parseMachO(str): # os, arch, bits, endianness arch = str.split(' ') arch = arch[2] bits = str.split(' ')[1].replace('-bit', '') - endianness = "lsb" + endianness = "little_endian" return [os, arch, bits, endianness] @@ -65,24 +67,28 @@ def parsePE32(str): bits = '64' if arch == '80386': arch = 'i386' - endianness = "lsb" + endianness = "little_endian" return [os, arch, bits, endianness] -def parseElfArch(str, architecture): +def parseElfArch(str, architecture, bits): architecture = architecture.strip() if architecture.startswith("ARM"): if 'aarch64' in architecture: - return 'aarch64' + return 'arm64' if 'armhf' in str: # this does not work for some reason - from_file() returns longer data than from_buffer() - needs fix - return 'armhf' + return 'arm' # because qt does not report it directly elif architecture.startswith("Intel"): if '80386' in architecture: return 'i386' elif architecture.startswith("IBM S/390"): return 's/390' elif "PowerPC" in architecture: - return 'powerpc' + if bits == "64": + return "power64" + else: + return 'power' + # sparc architecture is currently ignored (should be handled similar to power?) return architecture.lower() @@ -90,9 +96,15 @@ def parseElf(str): os = "Linux" arch = str.split(',') arch = arch[1] - arch = parseElfArch(str, arch) bits = str.split(' ')[1].replace('-bit', '') - endianness = str.split(' ')[2].lower() + arch = parseElfArch(str, arch, bits) + endian = str.split(' ')[2].lower() + if endian == "msb": + endianness = "big_endian" + elif endian == "lsb": + endianness = "little_endian" + else: + raise Exception("Unrecognised endianness") return [os, arch, bits, endianness] @@ -117,3 +129,26 @@ def getOsArch(str): if os: return result return None + +def normalizeArch(inputArch): + """ + This function brings requested architecture to common form (currently just parses the bits part and turns it into 32/64) + Input string format is: arch-endianness-word_size-kernelType + + """ + parts = inputArch.split('-') + #Drop anything non-numeric from word_size field + parts[2]=re.sub(r"\D", "", parts[2]) + #Transform kernelType into binary format + temp = parts[3] + if "win" in temp: + parts[3]="pe32" + elif "linux" in temp: + parts[3]="elf" + elif "freebsd" in temp: #How do we treat QNX? + parts[3]="elf" + elif "darwin" in temp: + parts[3]="mach_o" + #Rejoin new architecture + arch = '-'.join(parts) + return arch From 6e509735e47fcfb289f75863953aed6bb6396408 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 5 Dec 2018 17:01:09 +0300 Subject: [PATCH 05/48] Support for tag versions Support for versioned tags. Versioning is in form of either: tag or tag:version Task-number: AUTOSUITE-708 Change-Id: I480ae08b3cdc0ccf6ac1fc1c9724448be9cb1b55 Reviewed-by: Dominik Holland --- store/admin.py | 2 +- store/api.py | 29 +-- store/migrations/0001_initial.py | 2 +- store/models.py | 4 +- store/tags.py | 339 +++++++++++++++++++++++++++++++ store/utilities.py | 17 +- 6 files changed, 363 insertions(+), 30 deletions(-) create mode 100644 store/tags.py diff --git a/store/admin.py b/store/admin.py index 1cc06da..a921e2f 100644 --- a/store/admin.py +++ b/store/admin.py @@ -174,7 +174,7 @@ def save(self, commit=False): class AppAdmin(admin.ModelAdmin): form = AppAdminForm - list_display = ('name', 'appid', 'architecture', 'version') + list_display = ('name', 'appid', 'architecture', 'version', 'tags') def save_model(self, request, obj, form, change): obj.save() diff --git a/store/api.py b/store/api.py index 6e951a4..b9dc0e6 100644 --- a/store/api.py +++ b/store/api.py @@ -42,8 +42,8 @@ from models import App, Category, Vendor, savePackageFile from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage -from utilities import packagePath, iconPath, downloadPath, validateTag -from osandarch import normalizeArch +from utilities import packagePath, iconPath, downloadPath +from tags import SoftwareTagList def hello(request): @@ -58,12 +58,11 @@ def hello(request): for j in ("require_tag", "conflicts_tag",): if j in request.REQUEST: #Tags are coma-separated, - taglist = [i.lower() for i in request.REQUEST[j].split(',') if i] - for i in taglist: - if not validateTag(i): #Tags must be alphanumeric (or, even better - limited to ASCII alphanumeric) - status = 'malformed-tag' - break - request.session[j] = taglist + versionmap = SoftwareTagList() + if not versionmap.parse(request.REQUEST[j]): + status = 'malformed-tag' + break + request.session[j] = str(versionmap) if 'architecture' in request.REQUEST: request.session['architecture'] = normalizeArch(request.REQUEST['architecture']) else: @@ -169,13 +168,15 @@ def appList(request): #"require_tag", "conflicts_tag" # Tags are combined by logical AND (for require) and logical OR for conflicts if 'require_tag' in request.session: - for i in request.session['require_tag']: - regex = '(^|,)%s(,|$)' % (i,) - apps = apps.filter(Q(tags__regex = regex)) + require_tags = SoftwareTagList() + require_tags.parse(request.session['require_tag']) + for i in require_tags.make_regex(): + apps = apps.filter(Q(tags__regex = i)) if 'conflicts_tag' in request.session: - for i in request.session['conflicts_tag']: - regex = '(^|,)%s(,|$)' % (i,) - apps = apps.filter(~Q(tags__regex = regex)) + conflict_tags = SoftwareTagList() + conflict_tags.parse(request.session['conflicts_tag']) + for i in conflict_tags.make_regex(): + apps = apps.filter(~Q(tags__regex=i)) # Here goes the logic of listing packages when multiple architectures are available # in /hello request, the target architecture is stored in the session. By definition target machine can support diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index 772b1d2..ae4bd24 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -99,6 +99,6 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='app', - unique_together=set([('appid', 'architecture')]), + unique_together=set([('appid', 'architecture', 'tags')]), ), ] diff --git a/store/models.py b/store/models.py index 63b1d89..dd3b4a1 100644 --- a/store/models.py +++ b/store/models.py @@ -128,10 +128,10 @@ class App(models.Model): class Meta: """Makes the group of id and arch - a unique identifier""" - unique_together = (('appid', 'architecture', ),) + unique_together = (('appid', 'architecture', 'tags'),) def __unicode__(self): - return self.name + " [" + " ".join([self.appid,self.version,self.architecture]) + "]" + return self.name + " [" + " ".join([self.appid,self.version,self.architecture,self.tags]) + "]" def save(self, *args, **kwargs): try: diff --git a/store/tags.py b/store/tags.py new file mode 100644 index 0000000..183a90a --- /dev/null +++ b/store/tags.py @@ -0,0 +1,339 @@ +# vim: set fileencoding=utf-8 : +############################################################################# +## +## Copyright (C) 2018 Pelagicore AG +## Contact: https://www.qt.io/licensing/ +## +## This file is part of the Neptune Deployment Server +## +## $QT_BEGIN_LICENSE:GPL-QTAS$ +## Commercial License Usage +## Licensees holding valid commercial Qt Automotive Suite licenses may use +## this file in accordance with the commercial license agreement provided +## with the Software or, alternatively, in accordance with the terms +## contained in a written agreement between you and The Qt Company. For +## licensing terms and conditions see https://www.qt.io/terms-conditions. +## For further information use the contact form at https://www.qt.io/contact-us. +## +## GNU General Public License Usage +## Alternatively, this file may be used under the terms of the GNU +## General Public License version 3 or (at your option) any later version +## approved by the KDE Free Qt Foundation. The licenses are as published by +## the Free Software Foundation and appearing in the file LICENSE.GPL3 +## included in the packaging of this file. Please review the following +## information to ensure the GNU General Public License requirements will +## be met: https://www.gnu.org/licenses/gpl-3.0.html. +## +## $QT_END_LICENSE$ +## +## SPDX-License-Identifier: GPL-3.0 +## +############################################################################# + +import hashlib +import unittest +import re + + +def validateTagVersion(version): + for i in version: + if not i.isalnum(): + if not ((i == "_") or (i == ".")): + return False + return True + + +def validateTag(tag): + if len(tag) == 0: + return False + lst = tag.split(':') + if len(lst) > 2: + return False # More than one version component is not allowed + for i in lst[0]: + if not i.isalnum(): + if i != "_": + return False + if len(lst) > 1: + return validateTagVersion(lst[1]) + return True + + +class SoftwareTag: + def __init__(self, tag): + """ Takes tag and parses it. If it can't parse - raises exception of invalid value + :type tag: str + """ + if not ((type(tag) == str) or (type(tag) == unicode)): + raise (BaseException("Invalid input data-type")) + if not validateTag(tag): + raise (BaseException("Malformed tag")) + tag_version = tag.split(':') + self.tag = tag_version[0].lower() # No, this should be lowercase + self.version = None if len(tag_version) == 1 else tag_version[1] + + def __repr__(self): + return "SoftwareTag()" + + def __str__(self): + if self.version: + return "%s:%s" % (self.tag, self.version) + else: + return self.tag + + def has_version(self): + return self.version is not None + + def match(self, tag): # self is "on server", tag is "request" + if self.tag == tag.tag: + # Names are the same, that is good, matching versions now. + if self.version == tag.version: + return True + else: + if tag.version is None: + return True # qt in request, anything else on server - True + if self.version is not None and self.version.startswith(tag.version + "."): + return True + return False + return False + + def make_regex(self): + if self.version is None: + # versionless tag + temp_string = re.escape(self.tag) + regex = "(%s:[a-z0-9_.]*)|(%s)" % (temp_string, temp_string,) + else: + # tag with versions + temp_string = re.escape("%s:%s" % (self.tag, self.version)) + regex = "(%s\.[a-z0-9_.]*)|(%s)" % (temp_string, temp_string) + return regex + + +class SoftwareTagList: + def __init__(self): + # dictionary of tags, key is - tag name + self.taglist = dict() + + def __str__(self): + lst = list() + for key, value in self.taglist.items(): + lst += [str(i) for i in value] + lst.sort() + return ",".join(lst) + + def __repr__(self): + return "SoftwareTagList()" + + def __getitem__(self, item): + return self.taglist[item] + + def parse(self, tag_string): + self.taglist = dict() + try: + return all(self.append(SoftwareTag(i)) for i in tag_string.split(',')) + except: + return False + + def has_version(self, tag_name): + if tag_name in self.taglist: + # This check is possible, because, when there is tag without version - it is the only tag in the list + if self.taglist[tag_name][0].has_version(): + return True + return False + + def append(self, tag): + # tag should be SoftwareTag, return false or raise exception in case it is not so + if tag.has_version(): + if tag.tag in self.taglist: + # Tag in list - need to check version + if self.has_version(tag.tag) and not any(tag.match(i) for i in self.taglist[tag.tag]): + self.taglist[tag.tag].append(tag) + self.taglist[tag.tag].sort() # this is slow, I guess + else: + # Tag not in list - just add it. + self.taglist[tag.tag] = [tag, ] + else: + # tag without version tag + self.taglist[tag.tag] = [tag, ] + return True + + def is_empty(self): + return len(self.taglist) == 0 + + def make_regex(self): + lst = list() + for key, value in self.taglist.items(): + regex = "(^|,)%s(,|$)" % "|".join([i.make_regex() for i in value]) + lst.append(regex) + return lst + + def match_positive(self, taglist): + # checks that everything from tag list matches current tags + # Start checking with checking if all requested tags in taglist are present in self.taglist + for i in taglist.taglist: + if i not in self.taglist: + return False + # Now we need to check if versions are matching + for tag in taglist.taglist: + if not self.has_version(tag): + # If package tag accepts anything - it already matches, next please + continue + if taglist.has_version(tag) and not any(v1.match(v) for v in taglist[tag] for v1 in self[tag]): + return False + return True + + def match_negative(self, taglist): + # checks that nothing from taglist matches current tags + for i in taglist.taglist: + if i in self.taglist: + if (not taglist.has_version(i)) or (not self.has_version(i)): + return False + # Tag found, version list is present. check if it matches, if it does - check is failed + for version in taglist[i]: + for version1 in self[i]: + if version1.match(version): + return False + return True + + def hash(self): + # Looks like the list is sorted, but well... + return hashlib.md5(str(self)).hexdigest() + + +class TestSoftwareTagMethods(unittest.TestCase): + def test_tag_creation(self): + tag = SoftwareTag('qt') + self.assertFalse(tag.has_version()) + tag = SoftwareTag('qt:5.01') + self.assertTrue(tag.has_version()) + tag = SoftwareTag('qt:5.01_asdf_the_version') + self.assertTrue(tag.has_version()) + tag = SoftwareTag('Qt') + self.assertFalse(tag.has_version()) + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + SoftwareTag('фыва') + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + SoftwareTag('фыва:5.1') + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + SoftwareTag('qt.1:5.1') + with self.assertRaisesRegexp(BaseException, "Invalid input data-type"): + SoftwareTag(1) + + def test_tag_match(self): + tag_13 = SoftwareTag('qt:1.3') + tag_12 = SoftwareTag('qt:1.2') + tag_121 = SoftwareTag('qt:1.2.1') + tag_122 = SoftwareTag('qt:1.2.2') + tag = SoftwareTag('qt') + tag2 = SoftwareTag('neptune') + self.assertFalse(tag_12.match(tag_13)) + self.assertFalse(tag_13.match(tag_12)) + self.assertTrue(tag_121.match(tag_12)) + self.assertTrue(tag_122.match(tag_12)) + self.assertTrue(tag_121.match(tag_121)) + self.assertFalse(tag_12.match(tag_121)) + self.assertFalse(tag.match(tag2)) + self.assertTrue(tag_13.match(tag)) + self.assertFalse(tag.match(tag_13)) + + +class TestSoftwareTagListMethods(unittest.TestCase): + def test_empty(self): + lst = SoftwareTagList() + self.assertTrue(lst.is_empty()) + + def test_not_empty_after_append(self): + lst = SoftwareTagList() + lst.append(SoftwareTag('qt')) + self.assertFalse(lst.is_empty()) + + def test_empty_matches_everything(self): + empty_list = SoftwareTagList() + test_list = SoftwareTagList() + test_list.append(SoftwareTag('qt')) + self.assertTrue(test_list.match_positive(empty_list)) + self.assertTrue(test_list.match_negative(empty_list)) + + def test_match_positive(self): + list_to_test = SoftwareTagList() + list_to_test.parse("qt:5.1,neptune,test:1,second_test") + matching_list = SoftwareTagList() + matching_list.parse("qt") + self.assertTrue(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1") + self.assertTrue(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1,qt:5.2,neptune:1") + self.assertTrue(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1,test:2") + self.assertFalse(list_to_test.match_positive(matching_list)) + matching_list.parse("qt:5.1.1") + self.assertFalse(list_to_test.match_positive(matching_list)) + + def test_match_negative(self): + list_to_test = SoftwareTagList() + list_to_test.parse("qt:5.1,neptune") + matching_list = SoftwareTagList() + matching_list.parse("qt") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("qt:5.1") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("qt:5.1,qt:5.2,neptune:1") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("qt:5.1,qt:5.2") + self.assertFalse(list_to_test.match_negative(matching_list)) + matching_list.parse("test") + self.assertTrue(list_to_test.match_negative(matching_list)) + + def test_append_invalid(self): + lst = SoftwareTagList() + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + self.assertFalse(lst.append(SoftwareTag('qt:1:1'))) # Invalid version + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + self.assertFalse(lst.append(SoftwareTag('фыва'))) # Non-ascii + with self.assertRaisesRegexp(BaseException, "Malformed tag"): + self.assertFalse(lst.append(SoftwareTag(''))) # empty tag is not valid + + def test_append_valid(self): + lst = SoftwareTagList() + # capital letters should be treated as lowercase + self.assertTrue(lst.append(SoftwareTag('QT'))) + # underscore is allowed, capital letters should be treated as lowercase + self.assertTrue(lst.append(SoftwareTag('QT_something'))) + # Version is valid, tag is valid too + self.assertTrue(lst.append(SoftwareTag('qt:1.1.1'))) + + def test_parsing_positive(self): + lst = SoftwareTagList() + self.assertTrue(lst.parse('qt')) + self.assertTrue(lst.parse('qt:5')) + self.assertTrue(lst.parse('qt:5.1')) + self.assertTrue(lst.parse('qt:5.1,qt:5.2')) + self.assertTrue(lst.parse('qt:5.1,qt:5.2,neptune')) + self.assertTrue(lst.parse('qt:5.1,qt:5.2,neptune:5.1,neptune:5.2')) + # This should equal to qt:5.1,qt:5.2,neptune:5.1,neptune:5.2 - due to matching + self.assertTrue(lst.parse('qt:5.1,qt:5.2,qt:5.2,qt:5.2.1,neptune:5.1,neptune:5.2')) + # This equals to: qt, neptune, due to matching + self.assertTrue(lst.parse('qt,qt:5.2,neptune:5.1,neptune')) + + def test_parsing_negative(self): + lst = SoftwareTagList() + self.assertFalse(lst.parse(',,')) # empty tags + self.assertFalse(lst.parse('фыва')) # non-ascii + self.assertFalse(lst.parse('qt:5.1:5.2,qt')) # multiple versions + + def test_hashes_does_not_depend_on_order(self): + lst1 = SoftwareTagList() + lst2 = SoftwareTagList() + self.assertTrue(lst1.parse('qt:5,qt:4,neptune:1')) + self.assertTrue(lst2.parse('neptune:1,qt:4,qt:5')) + self.assertEqual(lst1.hash(), lst2.hash()) + + def test_different_list_different_hash(self): + lst1 = SoftwareTagList() + lst2 = SoftwareTagList() + self.assertTrue(lst1.parse('qt:5,neptune:2')) + self.assertTrue(lst2.parse('neptune:1,qt:5')) + self.assertNotEqual(lst1.hash(), lst2.hash()) + + +if __name__ == '__main__': + unittest.main() diff --git a/store/utilities.py b/store/utilities.py index bce9a7b..0f5465b 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -44,24 +44,17 @@ from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate from django.conf import settings +from tags import SoftwareTagList, SoftwareTag import osandarch -def validateTag(tag): - for i in tag: - if not i.isalnum(): - if i != "_": - return False - return True - def makeTagList(pkgdata): - taglist = set() + taglist = SoftwareTagList() for fields in ('extra', 'extraSigned'): if fields in pkgdata['header']: if 'tags' in pkgdata['header'][fields]: - tags = set(pkgdata['header'][fields]['tags']) # Fill tags list then add them - taglist = taglist.union(tags) - tags = ','.join(taglist) - return tags + for i in list(pkgdata['header'][fields]['tags']): # Fill tags list then add them + taglist.append(SoftwareTag(i)) + return str(taglist) def packagePath(appId = None, architecture = None): path = settings.MEDIA_ROOT + 'packages/' From 43b69a8dff2c0206c98d1cf33d47902a2e536a28 Mon Sep 17 00:00:00 2001 From: Dominik Holland Date: Thu, 13 Dec 2018 10:37:33 +0100 Subject: [PATCH 06/48] Qmake harness for building the documentation with qdoc Change-Id: I8ab555de921ce5ff655494d5e67dc10143242947 Reviewed-by: Robert Griebl --- .qmake.conf | 3 ++ doc/online/qtautodeploymentserver.qdocconf | 11 +++++++ doc/qtautodeploymentserver-project.qdocconf | 24 +++++++++++++++ doc/qtautodeploymentserver.qdocconf | 2 ++ doc/src/deployment-server.qdoc | 33 +++++++++++++++++++++ qtauto-deployment-server.pro | 13 ++++++++ 6 files changed, 86 insertions(+) create mode 100644 .qmake.conf create mode 100644 doc/online/qtautodeploymentserver.qdocconf create mode 100644 doc/qtautodeploymentserver-project.qdocconf create mode 100644 doc/qtautodeploymentserver.qdocconf create mode 100644 doc/src/deployment-server.qdoc create mode 100644 qtauto-deployment-server.pro diff --git a/.qmake.conf b/.qmake.conf new file mode 100644 index 0000000..9bdff44 --- /dev/null +++ b/.qmake.conf @@ -0,0 +1,3 @@ +MODULE_VERSION = 5.12 + +CONFIG += prepare_docs qt_docs_targets diff --git a/doc/online/qtautodeploymentserver.qdocconf b/doc/online/qtautodeploymentserver.qdocconf new file mode 100644 index 0000000..95a4f9e --- /dev/null +++ b/doc/online/qtautodeploymentserver.qdocconf @@ -0,0 +1,11 @@ +include($QT_INSTALL_DOCS/global/qt-module-defaults-online.qdocconf) + +# Add an .html file with sidebar content, used in the online style +# HTML.stylesheets += style/qt5-sidebar.html + +# override defaults for Qt module documentation +HTML.nosubdirs = "false" +HTML.outputsubdir = "qtauto-deployment-server" + +include(../qtautodeploymentserver-project.qdocconf) + diff --git a/doc/qtautodeploymentserver-project.qdocconf b/doc/qtautodeploymentserver-project.qdocconf new file mode 100644 index 0000000..65a4fd7 --- /dev/null +++ b/doc/qtautodeploymentserver-project.qdocconf @@ -0,0 +1,24 @@ +project = QtAutoDeploymentServer +description = Qt Auto Deployment Server Documentation +version = $QT_VERSION +url = http://doc.qt.io/QtAutoDeploymentServer + +sourcedirs += src +imagedirs += src/images + +qhp.projects = QtAutoDeploymentServer + +qhp.QtAutoDeploymentServer.file = qtautodeploymentserver.qhp +qhp.QtAutoDeploymentServer.namespace = org.qt-project.qtautodeploymentserver.$QT_VERSION_TAG +qhp.QtAutoDeploymentServer.virtualFolder = qtautodeploymentserver +qhp.QtAutoDeploymentServer.indexTitle = Qt Auto Deployment Server +qhp.QtAutoDeploymentServer.indexRoot = + +qhp.QtAutoDeploymentServer.filterAttributes = qtautodeploymentserver $QT_VERSION qtrefdoc +qhp.QtAutoDeploymentServer.customFilters.Qt.name = QtAutoDeploymentServer $QT_VERSION +qhp.QtAutoDeploymentServer.customFilters.Qt.filterAttributes = qtautodeploymentserver $QT_VERSION + +tagfile = qtautodeploymentserver.tags + +buildversion = "Qt Auto Deploymment Server $QT_VERSION" +navigation.homepage = "Qt Auto Deployment Server" diff --git a/doc/qtautodeploymentserver.qdocconf b/doc/qtautodeploymentserver.qdocconf new file mode 100644 index 0000000..fa27088 --- /dev/null +++ b/doc/qtautodeploymentserver.qdocconf @@ -0,0 +1,2 @@ +include($QT_INSTALL_DOCS/global/qt-module-defaults-offline.qdocconf) +include(qtautodeploymentserver-project.qdocconf) diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc new file mode 100644 index 0000000..3a8a7c9 --- /dev/null +++ b/doc/src/deployment-server.qdoc @@ -0,0 +1,33 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-index.html + \title Qt Auto Deployment Server + +*/ diff --git a/qtauto-deployment-server.pro b/qtauto-deployment-server.pro new file mode 100644 index 0000000..f7f9002 --- /dev/null +++ b/qtauto-deployment-server.pro @@ -0,0 +1,13 @@ +TEMPLATE = aux + +build_online_docs: { + QMAKE_DOCS = $$PWD/doc/online/qtautodeploymentserver.qdocconf +} else { + QMAKE_DOCS = $$PWD/doc/qtautodeploymentserver.qdocconf +} + +OTHER_FILES += \ + $$PWD/doc/*.qdocconf \ + $$PWD/doc/src/*.qdoc + +load(qt_docs) From 1a50e4e42689ba2211e145effdd779247957aa98 Mon Sep 17 00:00:00 2001 From: Robert Griebl Date: Thu, 13 Dec 2018 13:34:03 +0100 Subject: [PATCH 07/48] Update copyright statements - added 2019 - everything is (c) Luxoft instead of Pelagicore from now on Change-Id: If1b2f18f41dff7531e0c48329f9bf555192cbc2a Reviewed-by: Nikolay Zamotaev --- README.md | 2 +- appstore/settings.py | 3 ++- appstore/urls.py | 3 ++- appstore/wsgi.py | 3 ++- header.GPL-QTAS | 3 ++- manage.py | 3 ++- store/admin.py | 3 ++- store/api.py | 3 ++- store/authdecorators.py | 3 ++- store/management/commands/expire-downloads.py | 3 ++- store/management/commands/store-sign-package.py | 3 ++- store/management/commands/store-upload-package.py | 3 ++- store/management/commands/verify-upload-package.py | 3 ++- store/migrations/0001_initial.py | 3 ++- store/models.py | 3 ++- store/osandarch.py | 1 + store/tags.py | 1 + store/utilities.py | 3 ++- 18 files changed, 33 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ebd0327..28a473e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ This is a PoC deployment server, which can be used together with -the Neptune IVI UI and the Pelagicore Application Manager. +the Neptune IVI UI and the Luxoft Application Manager. **This is a development server only - do NOT use in production.** diff --git a/appstore/settings.py b/appstore/settings.py index 377843a..a405491 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/appstore/urls.py b/appstore/urls.py index 327c28b..4d0467f 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/appstore/wsgi.py b/appstore/wsgi.py index 9e80014..3f007d4 100644 --- a/appstore/wsgi.py +++ b/appstore/wsgi.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/header.GPL-QTAS b/header.GPL-QTAS index 0bd19ca..eb625be 100644 --- a/header.GPL-QTAS +++ b/header.GPL-QTAS @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/manage.py b/manage.py index bd587ad..ad7604e 100644 --- a/manage.py +++ b/manage.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/admin.py b/store/admin.py index a921e2f..eff2550 100644 --- a/store/admin.py +++ b/store/admin.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/api.py b/store/api.py index b9dc0e6..0ed54b1 100644 --- a/store/api.py +++ b/store/api.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/authdecorators.py b/store/authdecorators.py index fbbc01d..2a4119c 100644 --- a/store/authdecorators.py +++ b/store/authdecorators.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2018 Luxoft +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/expire-downloads.py b/store/management/commands/expire-downloads.py index edc5235..94b0d24 100644 --- a/store/management/commands/expire-downloads.py +++ b/store/management/commands/expire-downloads.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/store-sign-package.py b/store/management/commands/store-sign-package.py index 479b4c4..b1a42d0 100644 --- a/store/management/commands/store-sign-package.py +++ b/store/management/commands/store-sign-package.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 81f96fa..77286bc 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2018 Luxoft +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/management/commands/verify-upload-package.py b/store/management/commands/verify-upload-package.py index 87ac08a..59df116 100644 --- a/store/management/commands/verify-upload-package.py +++ b/store/management/commands/verify-upload-package.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index ae4bd24..51b0677 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/models.py b/store/models.py index dd3b4a1..eb3ede9 100644 --- a/store/models.py +++ b/store/models.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server diff --git a/store/osandarch.py b/store/osandarch.py index c7d15a6..9a2db12 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -1,5 +1,6 @@ ############################################################################# ## +## Copyright (C) 2019 Luxoft Sweden AB ## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## diff --git a/store/tags.py b/store/tags.py index 183a90a..aa73006 100644 --- a/store/tags.py +++ b/store/tags.py @@ -1,6 +1,7 @@ # vim: set fileencoding=utf-8 : ############################################################################# ## +## Copyright (C) 2019 Luxoft Sweden AB ## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## diff --git a/store/utilities.py b/store/utilities.py index 0f5465b..42f78f7 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -1,6 +1,7 @@ ############################################################################# ## -## Copyright (C) 2016 Pelagicore AG +## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## ## This file is part of the Neptune Deployment Server From 658e70fa01ad88cbfaaf166319c54a69792336e9 Mon Sep 17 00:00:00 2001 From: Jukka Jokiniva Date: Mon, 17 Dec 2018 15:02:26 +0200 Subject: [PATCH 08/48] Add missing header.FDL-QTAS template Change-Id: I092f279030a70586697b2947ec35653f1e938f0b Reviewed-by: Dominik Holland --- header.FDL-QTAS | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 header.FDL-QTAS diff --git a/header.FDL-QTAS b/header.FDL-QTAS new file mode 100644 index 0000000..6cac945 --- /dev/null +++ b/header.FDL-QTAS @@ -0,0 +1,27 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ From c31a12d694ed3862cb03f305b4a09c9e028ee299 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 14 Dec 2018 17:13:14 +0300 Subject: [PATCH 09/48] Documentation for qtauto deployment server First version. Change-Id: I44132467baedf41812e12f67170005d2508dec1e Fixes: AUTOSUITE-625 Reviewed-by: Svetlana Abramenkova --- doc/src/deployment-server.qdoc | 446 +++++++++++++++++++++++++++++++++ 1 file changed, 446 insertions(+) diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 3a8a7c9..c505c9b 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -29,5 +29,451 @@ /*! \page qtauto-deployment-server-index.html \title Qt Auto Deployment Server + \section1 Introduction + Qt Automotive Suite Deployment server is a reference implementation of a network resource for + hosting and distributing applications in projects based on Qt Application Manager. + Even though the Deployment Server can be used in field for real products, the main purpose + for it is helping development and integration teams to simplify development and testing of + applications on target system. + \section1 Assumptions + \list + \li Applications are identified by a group of (Application ID, Version, Architecture, Tags). + Such groups are unique. + \li Architecture is specified as a group of: (CPU architecture, endianness, bitness and OS). + If package does not contain architecture specific parts, architecture is specified as “All”. + \li CPU architecture is taken as returned by QsysInfo::buildCpuArchitecture(). + \li As it is hard to determine OS for uploaded packages, OS-es are matched by binary format, + requesting anything unix-like will give packages with ELF binaries, anything Windows related + will return packages with PE32 format binaries and packages with Mach_o binaries are returned + for Apple products. See \l {http://doc.qt.io/qt-5/qsysinfo.html#kernelType} {QSysInfo::kernelType()} for more details. + \li When native and non-native applications match selection criteria, native application is preferred. + \li Applications can be further filtered by tags, both as positive (must have) and negative + (must not have) filters. + \li Tags are considered alphanumeric and can contain lowercase latin letters, numbers and + underscore symbol. All tags passed to the server are brought to lowercase. + \li Tags can optionally have version. Version is separated from tag by ':'. Version follows the same + limitations as tag. + \li Tags are matched according to versions (when requesting version 5.12, it will match 5.12.0, but not vice versa). + Also, when requesting non-versioned tag - any version will match. + \li Tag lists in requests and packages are simplified (so qt:5.12,qt:5.12.0 will end up as qt:5.12) + \li \l {https://doc.qt.io/QtApplicationManager/manifest.html#basic-manifest} {Application manifest} + allows for any number of categories assigned to application. Deployment server currently + ignores categories in application manifest and requires manual assignment of exactly one + category to the application. This behavior can be discussed with nzamotaev@luxoft.com . + \li Tag information is parsed from package header's \b{extra} and \b{extraSigned} parts, + from \b{tags} array. All elements of that array are added to package’s tag list. + \li Every package has version. If manifest does not contain version, + it will be assigned version “0.0.0”. + \endlist + + \section1 API + \section2 hello + Checks whether you are using the right Platform and the right API to communicate with the Server. + \table + \header + \li Parameter + \li Description + \row + \li platform + \li The platform the client is running on, this sets the architecture of the packages you get. (see \b{settings.APPSTORE_PLATFORM}) + \row + \li version + \li The Deployment Server HTTP API version you are using to communicate with the server. (see \b{settings.APPSTORE_VERSION}) + \row + \li require_tag + \li Optional parameter for filtering packages by tags. Receives comma-separated list of tags. + Only applications containing any of specified tags should be listed. Tags must be alphanumeric. + \row + \li conflicts_tag + \li Optional parameter for filtering packages by tags. Receives comma-separated list of tags. + No application containing any of the tags should be listed. Tags must be alphanumeric. + \row + \li architecture + \li ptional parameter for filtering packages by architecture. Receives cpu architecture. + If architecture is not specified, only packages showing 'All' architecture are listed. + \endtable + Returns a JSON object: + \table + \header + \li JSON field + \li Value + \li Description + \row + \li {1,5} status + \li ok + \li Successful + \row + \li maintenance + \li Server is in maintenance mode and can not be used at the moment. + \row + \li incompatible-platform + \li You are using incompatible platform. + \row + \li incompatible-version + \li You are using incompatible version of the API + \row + \li malformed-tag + \li Tag had wrong format, was not alphanumeric or could not be parsed. + \endtable + \section2 login + Does a login on the Server with the given username and password. + Either a IMEI or \l {https://en.wikipedia.org/wiki/MAC_address} {other unique hardware identifier} must be provided. This call is needed for downloading apps. + \table + \header + \li Parameter + \li Description + \row + \li username + \li The username + \row + \li password + \li Password for given username + \endtable + Returns a JSON object: + \table + \header + \li JSON field + \li Value + \li Description + \row + \li {1,4} status + \li ok + \li Login successful. + \row + \li missing-credentials + \li No username and/or password provided + \row + \li account-disabled + \li Account is disabled in admin panel + \row + \li authentication-failed + \li Wrong username and/or password or other authentication error + \endtable + \section2 logout + Logs out currently logged in user on server. + + Returns a JSON object: + \table + \header + \li JSON field + \li Value + \li Description + \row + \li {1,2} status + \li ok + \li Successfully logged out + \row + \li failed + \li Was not logged in + \endtable + + \section2 app/list + Lists all apps. Returned List can be filtered by using the \b{category_id} and the \b{filter} arguments. + \table + \header + \li Parameter + \li Description + \row + \li category_id + \li Limits applications to category with this id. + \row + \li filter + \li Only lists apps, whose names match the filter. + \endtable + Returns a JSON array (\b{not an object!}). Each field is a JSON object: + \table + \header + \li JSON field + \li Description + \row + \li id + \li Unique application id. In reverse domain name notation + \row + \li name + \li Application name + \row + \li vendor + \li Vendor name for application (not vendor id) + \row + \li category + \li Category name for application + \row + \li tags + \li JSON array of application tags + \row + \li version + \li Application version. Returned as string. If application information lacks version, “0.0.0” is returned + \row + \li architecture + \li Application architecture. Returned as detected in library components of the application + + If application is not native, contains “All”. Otherwise it is formed like this: mips-little_endian-32-elf + + Where it is a combination of: + \list 1 + \li CPU architecture, as returned by QsysInfo::buildCpuArchitecture() + \li CPU endianness (either \b{little_endian} or \b{big_endian}) + \li ABI bitness + \li binary format (\b{elf}, \b{mach_o} or \b{pe32}) + \endlist + + \row + \li briefDescription + \li Short textual app description (should be limited to 1 line, around 80-130 characters) + \row + \li category_id + \li Numeric category id matching application category field. + \endtable + \section2 app/icon + Returns an icon for the given application id. + \table + \header + \li Parameter + \li Description + \row + \li id + \li Application id + \endtable + Returns a PNG image or a 404 error, if application does not exist + \section2 app/description + Returns a description for the given application id. + \table + \header + \li Parameter + \li Description + \row + \li id + \li Application id + \endtable + Returns application description text - either HTML or plain text. + \section2 app/purchase + Returns an url which can be used for downloading the requested application for certain period of time (configurable in the settings). + \note This is legacy from AppStore, changing the name of this API would involve changes in reference UI. + The real action is preparing package for download. For discussion, contact nzamotaev@luxoft.com. + \table + \header + \li Parameter + \li Description + \row + \li device_id + \li Unique device id for client hardware (currently unused) + \row + \li id + \li Application id + \endtable + Returns a JSON object: + \table + \header + \li JSON field + \li Value + \li Description + \row + \li {1,2} status + \li ok + \li Successful + \row + \li failed + \li Something went wrong. Check error field for more information. + \row + \li error + \li text + \li If status equals to ‘failed’, contains error description. + \row + \li url + \li URL + \li URL for downloading application. Expires in time specified in ‘expiresIn’ field. + \row + \li expiresIn + \li Integer value + \li Time in seconds for which download url stays valid. + \endtable + + \section2 category/list + Lists all the available categories. Also returns ‘All’ metacategory, that is used to hold all available applications. + Returns a JSON array (not an object!). Each field is a JSON object: + \table + \header + \li JSON field + \li Description + \row + \li id + \li Unique category id. + \row + \li name + \li Category name. + \endtable + \section2 category/icon + Returns an icon for the given category id. + \table + \header + \li Parameter + \li Description + \row + \li id + \li Id of the category. + \endtable + Returns a PNG image or empty 1x1 PNG file. + \note Currently takes icon of the first application in the category if it exists. This should be fixed. + \section2 upload + Accepts remote package upload requests. + User must be in the “staff” group to use this API. Also requires either basic authentication, + or previous call to \b{login} method. This is a POST request to server, due to parameters used. + \table + \header + \li Parameter + \li Description + \row + \li description + \li Package description, long version. Can be text or HTML. + \row + \li short-description + \li One line package description. + \row + \li category + \li Category name for the category where the package will be put. + \row + \li vendor + \li Vendor name for the package. + \row + \li package + \li Package itself. This is uploaded as a file parameter. + \endtable + Returns JSON object: + \table + \header + \li Parameter + \li Value + \li Description + \row + \li {1,9} status + \li ok + \li Success + \row + \li no description + \li Description parameter missing + \row + \li no short description + \li Short-description parameter missing + \row + \li no category + \li Category parameter missing + \row + \li no vendor + \li Vendor parameter missing + \row + \li Package validation failed + \li Package did not pass format/sanity validation + \row + \li Non-existing category + \li No category matches passed parameter + \row + \li Non-existing vendor + \li No vendor matches passed parameter + \row + \li no package to upload + \li There was no ‘package’ parameter in request, or it was not a POST request + \endtable + + \section2 API use examples + The deployment server exposes a HTTP API to the world. Arguments to the functions need to be provided using the + HTTP GET/POST syntax. Returned data will be in JSON, PNG or text format, depending on the function. + \section3 Basic workflow + \list 1 + \li Send a \b{hello} request to the server to get the current status and check + whether your platform is compatible with this server instance: + + \tt http:///hello?platform=AM&version=1 + + Returns: + + \tt { { "status": "ok" } } + + \li Login as user \b{user} with password \b{pass}: + + \tt http:///login?username=user&password=pass + + Returns: + + \tt { { "status": "ok" } } + + \li List all applications + + \tt http:///app/list + + Returns: + + \tt { [{ "category": "Entertainment", + "name": "Nice App", + "vendor": "Luxoft", + "briefDescription": "Nice App is a really nice app.", + "category_id": 4, + "id": "com.luxoft.niceapp"}, + .... + ] } + + + \li Request a download for a App: + + \tt http:///app/purchase?device_id=12345&id=com.luxoft.niceapp + + Returns: + + \tt { { "status": "ok", + "url": "http:///app/download/com.luxoft.niceapp.2.npkg", + "expiresIn": 600 + } } + + \li Use the \b{url} provided in step 4 to download the application within + \b{expiresIn} seconds. + + \endlist + + \section1 Installation + \section2 Setting up the server in virtualenv + \code + virtualenv ./venv + ./venv/bin/pip install -r requirements.txt + \endcode + (libffi-dev is also needed)a + + Before running the server, make sure to adapt the \b{APPSTORE_*} settings in \b{appstore/settings.py} to your environment. + + Since package downloads are done via temporary files, one needs to setup a cron-job to cleanup + these temporary files every now and then. The job should be triggered every \b{settings.APPSTORE_DOWNLOAD_EXPIRY/2} + minutes and it just needs to execute: + \code + ./manage.py expire-downloads + \endcode + \section2 Starting the server + Running the server: + \code + ./manage.py runserver 0.0.0.0:8080 + \endcode + will start the server on port 8080, reachable for anyone. One can tweak the listening address to whatever fits the needs. + \section2 Maintenance tasks + \list + \li Cleaning up the downloads directory: + \code + ./manage.py expire-downloads + \endcode + will remove all files from the downloads/ directory, that are older than + \b{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. This should be called from a cron-job (see above). + \li Manually verifying a package for upload: + \code + ./manage.py verify-upload-package + \endcode + will tell if \b{} is a valid package that can be uploaded to the store. + \li Manually adding a store signature to a package: + \code + ./manage.py store-sign-package [device id] + \endcode + will first verify \b{}. If this succeeds, it will copy \b{} to \b{} and + add a store signature. The optional \b{[device id]} parameter will lock the generated package to the device with this id. + \endlist + + \section1 Architecture + This is a django application. + + \b{TBD} */ From 55dfdca96b04ba0e54ca16001a6411fd5f48d4ad Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 18 Dec 2018 15:03:15 +0300 Subject: [PATCH 10/48] Qtauto deployment server documentation update Fixes: AUTOSUITE-625 Change-Id: I5433ce0c2af64109d3e4d3779d698f2c93eadb28 Reviewed-by: Svetlana Abramenkova --- doc/src/deployment-server.qdoc | 141 ++++++++++++++++++--------------- 1 file changed, 75 insertions(+), 66 deletions(-) diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index c505c9b..1710ea4 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -28,34 +28,35 @@ /*! \page qtauto-deployment-server-index.html - \title Qt Auto Deployment Server - \section1 Introduction + \title Qt Automotive Suite Deployment Server Qt Automotive Suite Deployment server is a reference implementation of a network resource for hosting and distributing applications in projects based on Qt Application Manager. - Even though the Deployment Server can be used in field for real products, the main purpose + Even though the deployment server can be used in the field for real products, the main purpose for it is helping development and integration teams to simplify development and testing of applications on target system. + + When implementing the deployment server, certain assumptions were made. \section1 Assumptions \list - \li Applications are identified by a group of (Application ID, Version, Architecture, Tags). + \li Applications are identified by a group of: Application ID, version, architecture and tags. Such groups are unique. - \li Architecture is specified as a group of: (CPU architecture, endianness, bitness and OS). - If package does not contain architecture specific parts, architecture is specified as “All”. + \li Architecture is specified as a group of: CPU architecture, endianness, bitness and OS. + If a package does not contain architecture specific parts, the architecture is specified as \e{All}. \li CPU architecture is taken as returned by QsysInfo::buildCpuArchitecture(). - \li As it is hard to determine OS for uploaded packages, OS-es are matched by binary format, - requesting anything unix-like will give packages with ELF binaries, anything Windows related + \li As it is hard to determine the OS for uploaded packages, they are matched by binary format. + Thus, requesting anything Unix-like will give packages with ELF binaries, anything Windows related will return packages with PE32 format binaries and packages with Mach_o binaries are returned for Apple products. See \l {http://doc.qt.io/qt-5/qsysinfo.html#kernelType} {QSysInfo::kernelType()} for more details. - \li When native and non-native applications match selection criteria, native application is preferred. + \li When native and non-native applications match selection criteria, the native application is preferred. \li Applications can be further filtered by tags, both as positive (must have) and negative (must not have) filters. - \li Tags are considered alphanumeric and can contain lowercase latin letters, numbers and - underscore symbol. All tags passed to the server are brought to lowercase. - \li Tags can optionally have version. Version is separated from tag by ':'. Version follows the same - limitations as tag. + \li Tags are considered alphanumeric and can contain lowercase Latin letters, numbers and + the underscore symbol. All tags passed to the server are converted to lowercase. + \li Tags can optionally have a version. The version number is separated from tag by a colon (:). The version follows the same + limitations as the tag. \li Tags are matched according to versions (when requesting version 5.12, it will match 5.12.0, but not vice versa). - Also, when requesting non-versioned tag - any version will match. + Also, when requesting a non-versioned tag - any version will match. \li Tag lists in requests and packages are simplified (so qt:5.12,qt:5.12.0 will end up as qt:5.12) \li \l {https://doc.qt.io/QtApplicationManager/manifest.html#basic-manifest} {Application manifest} allows for any number of categories assigned to application. Deployment server currently @@ -63,35 +64,38 @@ category to the application. This behavior can be discussed with nzamotaev@luxoft.com . \li Tag information is parsed from package header's \b{extra} and \b{extraSigned} parts, from \b{tags} array. All elements of that array are added to package’s tag list. - \li Every package has version. If manifest does not contain version, - it will be assigned version “0.0.0”. + \li Every package has a version number. If the manifest does not contain a version field, + the version number will be assigned “0.0.0”. \endlist \section1 API \section2 hello - Checks whether you are using the right Platform and the right API to communicate with the Server. + Checks whether you are using the correct Platform and the right API to communicate with the deployment server. \table \header \li Parameter \li Description \row \li platform - \li The platform the client is running on, this sets the architecture of the packages you get. (see \b{settings.APPSTORE_PLATFORM}) + \li The platform the client is running on. This sets the architecture of the packages + you get (see \b{settings.APPSTORE_PLATFORM} parameter in \b{appstore/settings.py} file). \row \li version - \li The Deployment Server HTTP API version you are using to communicate with the server. (see \b{settings.APPSTORE_VERSION}) + \li The Deployment Server HTTP API version you are using to communicate with the server + (see \b{settings.APPSTORE_VERSION}). \row \li require_tag - \li Optional parameter for filtering packages by tags. Receives comma-separated list of tags. - Only applications containing any of specified tags should be listed. Tags must be alphanumeric. + \li Optional parameter for filtering packages by tags. Receives a comma-separated list of tags. + Only applications containing any of the specified tags should be listed. + Tags must be alphanumeric. \row \li conflicts_tag - \li Optional parameter for filtering packages by tags. Receives comma-separated list of tags. + \li Optional parameter for filtering packages by tags. Receives a comma-separated list of tags. No application containing any of the tags should be listed. Tags must be alphanumeric. \row \li architecture - \li ptional parameter for filtering packages by architecture. Receives cpu architecture. - If architecture is not specified, only packages showing 'All' architecture are listed. + \li Optional parameter for filtering packages by architecture. Receives the CPU architecture. + If architecture is not specified, only packages showing \e{All} architecture are listed. \endtable Returns a JSON object: \table @@ -108,17 +112,17 @@ \li Server is in maintenance mode and can not be used at the moment. \row \li incompatible-platform - \li You are using incompatible platform. + \li You are using an incompatible platform. \row \li incompatible-version - \li You are using incompatible version of the API + \li You are using incompatible version of the API. \row \li malformed-tag \li Tag had wrong format, was not alphanumeric or could not be parsed. \endtable \section2 login - Does a login on the Server with the given username and password. - Either a IMEI or \l {https://en.wikipedia.org/wiki/MAC_address} {other unique hardware identifier} must be provided. This call is needed for downloading apps. + Logs onto the deployment server with the given username and password. + Either an IMEI or a \l {https://en.wikipedia.org/wiki/MAC_address} {unique hardware identifier} must be provided. This call is needed for downloading apps. \table \header \li Parameter @@ -128,7 +132,7 @@ \li The username \row \li password - \li Password for given username + \li Password for the given username \endtable Returns a JSON object: \table @@ -142,16 +146,16 @@ \li Login successful. \row \li missing-credentials - \li No username and/or password provided + \li No username or password provided \row \li account-disabled - \li Account is disabled in admin panel + \li Account is disabled in the admin panel \row \li authentication-failed \li Wrong username and/or password or other authentication error \endtable \section2 logout - Logs out currently logged in user on server. + Logs out the currently logged-in user from the deployment server. Returns a JSON object: \table @@ -169,19 +173,19 @@ \endtable \section2 app/list - Lists all apps. Returned List can be filtered by using the \b{category_id} and the \b{filter} arguments. + Lists all apps. The returned list can be filtered by using the \b{category_id} and the \b{filter} arguments. \table \header \li Parameter \li Description \row \li category_id - \li Limits applications to category with this id. + \li Limits applications to the category with this id. \row \li filter - \li Only lists apps, whose names match the filter. + \li Only lists apps whose names match the filter. \endtable - Returns a JSON array (\b{not an object!}). Each field is a JSON object: + Returns an array of JSON objects (\b{not an object itself!}). \table \header \li JSON field @@ -203,12 +207,14 @@ \li JSON array of application tags \row \li version - \li Application version. Returned as string. If application information lacks version, “0.0.0” is returned + \li Application version. Returned as a string. If the application information lacks + a version number, “0.0.0” is returned \row \li architecture - \li Application architecture. Returned as detected in library components of the application + \li Application architecture. Returned as detected in the library components of the application - If application is not native, contains “All”. Otherwise it is formed like this: mips-little_endian-32-elf + If the application is not native, contains \e{All}. + Otherwise it is formed like this: mips-little_endian-32-elf Where it is a combination of: \list 1 @@ -223,7 +229,7 @@ \li Short textual app description (should be limited to 1 line, around 80-130 characters) \row \li category_id - \li Numeric category id matching application category field. + \li Numeric category id matching the application category field. \endtable \section2 app/icon Returns an icon for the given application id. @@ -235,7 +241,7 @@ \li id \li Application id \endtable - Returns a PNG image or a 404 error, if application does not exist + Returns a PNG image or a 404 error, if the application does not exist. \section2 app/description Returns a description for the given application id. \table @@ -246,11 +252,11 @@ \li id \li Application id \endtable - Returns application description text - either HTML or plain text. + Returns an application description text - either HTML or plain text. \section2 app/purchase - Returns an url which can be used for downloading the requested application for certain period of time (configurable in the settings). - \note This is legacy from AppStore, changing the name of this API would involve changes in reference UI. - The real action is preparing package for download. For discussion, contact nzamotaev@luxoft.com. + Returns a URL which can be used for downloading the requested application for certain period of time (configurable in the settings). + \note This is legacy from AppStore. Changing the name of this API would involve changes in reference UI. + The real action is preparing the package for download. For discussion, contact nzamotaev@luxoft.com. \table \header \li Parameter @@ -274,24 +280,25 @@ \li Successful \row \li failed - \li Something went wrong. Check error field for more information. + \li Something went wrong. Check the error field for more information. \row \li error \li text - \li If status equals to ‘failed’, contains error description. + \li If status equals to \e{failed}, contains an error description. \row \li url \li URL - \li URL for downloading application. Expires in time specified in ‘expiresIn’ field. + \li URL for downloading the application. + Expires in the time specified in the \c{expiresIn} field. \row \li expiresIn \li Integer value - \li Time in seconds for which download url stays valid. + \li Time in seconds for which the download URL stays valid. \endtable \section2 category/list - Lists all the available categories. Also returns ‘All’ metacategory, that is used to hold all available applications. - Returns a JSON array (not an object!). Each field is a JSON object: + Lists all the available categories. Also returns \e{All} metacategory, that is used to hold all available applications. + Returns an array of JSON objects (\b{not an object itself!}). \table \header \li JSON field @@ -313,12 +320,13 @@ \li id \li Id of the category. \endtable - Returns a PNG image or empty 1x1 PNG file. - \note Currently takes icon of the first application in the category if it exists. This should be fixed. + Returns a PNG image or an empty 1x1 PNG file. + \note Currently takes the icon of the first application in the category if it exists. + This should be fixed. \section2 upload Accepts remote package upload requests. - User must be in the “staff” group to use this API. Also requires either basic authentication, - or previous call to \b{login} method. This is a POST request to server, due to parameters used. + The user must be in the \e{staff} group to use this API. Also requires either basic authentication + or a previous call to the \c{login} method. This is a POST request to the server due to the parameters used. \table \header \li Parameter @@ -363,24 +371,24 @@ \li Vendor parameter missing \row \li Package validation failed - \li Package did not pass format/sanity validation + \li Package did not pass format or sanity validation \row \li Non-existing category - \li No category matches passed parameter + \li No category matches the passed parameter \row \li Non-existing vendor \li No vendor matches passed parameter \row \li no package to upload - \li There was no ‘package’ parameter in request, or it was not a POST request + \li There was no \c{package} parameter in the request, or it was not a POST request \endtable \section2 API use examples The deployment server exposes a HTTP API to the world. Arguments to the functions need to be provided using the HTTP GET/POST syntax. Returned data will be in JSON, PNG or text format, depending on the function. - \section3 Basic workflow + \section3 Workflow \list 1 - \li Send a \b{hello} request to the server to get the current status and check + \li Send a \b{hello} request to the server to get the current status and to check whether your platform is compatible with this server instance: \tt http:///hello?platform=AM&version=1 @@ -389,7 +397,7 @@ \tt { { "status": "ok" } } - \li Login as user \b{user} with password \b{pass}: + \li Login as user \e{user} with password \e{pass}: \tt http:///login?username=user&password=pass @@ -413,7 +421,7 @@ ] } - \li Request a download for a App: + \li Request a download for an app: \tt http:///app/purchase?device_id=12345&id=com.luxoft.niceapp @@ -430,12 +438,13 @@ \endlist \section1 Installation - \section2 Setting up the server in virtualenv + \section2 Setting up the Server in virtualenv + Before installing dependencies in the Python virtual environment, libffi-dev package must be + installed. After package installation, prepare the virtual environment: \code virtualenv ./venv ./venv/bin/pip install -r requirements.txt \endcode - (libffi-dev is also needed)a Before running the server, make sure to adapt the \b{APPSTORE_*} settings in \b{appstore/settings.py} to your environment. @@ -450,14 +459,14 @@ \code ./manage.py runserver 0.0.0.0:8080 \endcode - will start the server on port 8080, reachable for anyone. One can tweak the listening address to whatever fits the needs. + will start the server on port 8080, reachable by anyone. One can tweak the listening address to whatever fits the needs. \section2 Maintenance tasks \list \li Cleaning up the downloads directory: \code ./manage.py expire-downloads \endcode - will remove all files from the downloads/ directory, that are older than + will remove all files from the downloads directory, that are older than \b{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. This should be called from a cron-job (see above). \li Manually verifying a package for upload: \code From 43ae22359264e89ee0383ed8d52032394758929c Mon Sep 17 00:00:00 2001 From: Topi Reinio Date: Thu, 20 Dec 2018 14:50:29 +0100 Subject: [PATCH 11/48] Doc: Fix documentation configuration - Fix references to landing page title - Fix typo in buildversion - Use https over http - Use 'Qt Automotive Suite' as navigation.homepage and add a dependency to it Task-number: AUTOSUITE-726 Change-Id: I48b47b872d231689387de1a48cc0cfc1dca76eec Reviewed-by: Dominik Holland --- doc/qtautodeploymentserver-project.qdocconf | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/doc/qtautodeploymentserver-project.qdocconf b/doc/qtautodeploymentserver-project.qdocconf index 65a4fd7..8625f13 100644 --- a/doc/qtautodeploymentserver-project.qdocconf +++ b/doc/qtautodeploymentserver-project.qdocconf @@ -1,7 +1,7 @@ project = QtAutoDeploymentServer -description = Qt Auto Deployment Server Documentation +description = Qt Automotive Suite Deployment Server Documentation version = $QT_VERSION -url = http://doc.qt.io/QtAutoDeploymentServer +url = https://doc.qt.io/QtAutoDeploymentServer sourcedirs += src imagedirs += src/images @@ -11,7 +11,7 @@ qhp.projects = QtAutoDeploymentServer qhp.QtAutoDeploymentServer.file = qtautodeploymentserver.qhp qhp.QtAutoDeploymentServer.namespace = org.qt-project.qtautodeploymentserver.$QT_VERSION_TAG qhp.QtAutoDeploymentServer.virtualFolder = qtautodeploymentserver -qhp.QtAutoDeploymentServer.indexTitle = Qt Auto Deployment Server +qhp.QtAutoDeploymentServer.indexTitle = Qt Automotive Suite Deployment Server qhp.QtAutoDeploymentServer.indexRoot = qhp.QtAutoDeploymentServer.filterAttributes = qtautodeploymentserver $QT_VERSION qtrefdoc @@ -20,5 +20,7 @@ qhp.QtAutoDeploymentServer.customFilters.Qt.filterAttributes = qtautodeployments tagfile = qtautodeploymentserver.tags -buildversion = "Qt Auto Deploymment Server $QT_VERSION" -navigation.homepage = "Qt Auto Deployment Server" +depends = qtautomotivesuite + +buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" +navigation.homepage = "Qt Automotive Suite" From 1fde901863d0ebf188360b781668c2199c32acdc Mon Sep 17 00:00:00 2001 From: Samuli Piippo Date: Mon, 4 Feb 2019 10:29:15 +0200 Subject: [PATCH 12/48] Add .tag file to record sha1 to the source package Task-number: AUTOSUITE-760 Change-Id: Iafd1894b97986ab1b405cd32fdbba32888b8f90f Reviewed-by: Robert Griebl --- .gitattributes | 1 + .tag | 1 + 2 files changed, 2 insertions(+) create mode 100644 .gitattributes create mode 100644 .tag diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5b4275b --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.tag export-subst diff --git a/.tag b/.tag new file mode 100644 index 0000000..6828f88 --- /dev/null +++ b/.tag @@ -0,0 +1 @@ +$Format:%H$ From 16c2655046c24805cf37df11a6795497514bb118 Mon Sep 17 00:00:00 2001 From: Kavindra Palaraja Date: Tue, 5 Feb 2019 17:08:00 +0100 Subject: [PATCH 13/48] [docs] - Update the overview on the Deployment Server - Added a new diagram based on input from Dominik, Robert, and Vladimir. - This change uses the term "Downloads app" which is currently not in sync with Neptune 3 UI's documentation. A subsequent patch will be made there to change "App store" to "Downloads app" Change-Id: I3fa05f6609420744f7c5cb570be91116207de4d1 Reviewed-by: Bramastyo Harimukti Santoso --- doc/src/deployment-server.qdoc | 83 +++++++++++------- ...ployment-server-installation-handshake.png | Bin 0 -> 35810 bytes 2 files changed, 53 insertions(+), 30 deletions(-) create mode 100644 doc/src/images/deployment-server-installation-handshake.png diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 1710ea4..d3f2e81 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -29,43 +29,66 @@ /*! \page qtauto-deployment-server-index.html \title Qt Automotive Suite Deployment Server - Qt Automotive Suite Deployment server is a reference implementation of a network resource for - hosting and distributing applications in projects based on Qt Application Manager. - Even though the deployment server can be used in the field for real products, the main purpose - for it is helping development and integration teams to simplify development and testing of - applications on target system. - When implementing the deployment server, certain assumptions were made. + The Qt Automotive Suite Deployment Server is a new component in the Qt Automotive Suite 5.12. + Previously, it was known as the Neptune Appstore and used for demonstrations purposes. + + This component is a lightweight HTTP server that provides network feeds to application packages + that are available for installation by a Qt Application Manager instance, running on a target + device connected to a network. The UI counterpart for the Deployment Server is the Downloads app + in the Neptune 3 UI. Together, the Deployment Server and the Downloads app enable you to install + different apps that are available in the server via network. + + The key scenario for these components is to install apps during the integration testing + process. Additionally, the code can also be used as a reference implementation for a + fully-featured server, and a new Downloads app on the target device for production. + + The Deployment Server works with the Application Installer in the Qt Application Manager and + acts as an installation source for \c{http://} and \c{https://} schemes. In addition to + application packages, the Deployment Server also hosts meta information about each package, + that is used by the Downloads app to visualize the choices available for the user to select. + This meta information is stored in the form of tags and other information in the package + header, according to the Qt Application Manager’s package format. When a package is uploaded + to the server, the package header is parsed, associated with that package, and then sent to a + Downloads app, that queries for a list of available apps. Using this information, a Downloads + app can inform users about apps that are available and even hide those that are not compatible + with the target installation. The figure below illustrates this installation scenario. + + \image deployment-server-installation-handshake.png "Install an App via the Deployment Server" + + The Deployment Server is implemented in Python, using Django, based on the following assumptions. + \section1 Assumptions \list - \li Applications are identified by a group of: Application ID, version, architecture and tags. - Such groups are unique. - \li Architecture is specified as a group of: CPU architecture, endianness, bitness and OS. + \li Applications are identified with a group of: Application ID, version, architecture, and tags; + these groups are unique. + \li Architecture is specified as a group of: CPU architecture, endianness, bitness, and OS. If a package does not contain architecture specific parts, the architecture is specified as \e{All}. - \li CPU architecture is taken as returned by QsysInfo::buildCpuArchitecture(). - \li As it is hard to determine the OS for uploaded packages, they are matched by binary format. - Thus, requesting anything Unix-like will give packages with ELF binaries, anything Windows related - will return packages with PE32 format binaries and packages with Mach_o binaries are returned - for Apple products. See \l {http://doc.qt.io/qt-5/qsysinfo.html#kernelType} {QSysInfo::kernelType()} for more details. - \li When native and non-native applications match selection criteria, the native application is preferred. - \li Applications can be further filtered by tags, both as positive (must have) and negative - (must not have) filters. - \li Tags are considered alphanumeric and can contain lowercase Latin letters, numbers and + \li CPU architecture is based on the return value from QsysInfo::buildCpuArchitecture(). + \li The installation target is automatically determined by parsing binary files. For example, + detecting an ELF binary means that it's a Linux package; the CPU architecture, such as armv8, + further defines the installation target. + See \l {https://doc.qt.io/qt-5/qsysinfo.html#kernelType} {QSysInfo::kernelType()} for more details. + \li If both native and non-native applications match the selection criteria, then the native application + is preferred. + \li Applications can be further filtered by tags, both as positive (inlucde) and negative (exclude) + filters. + \li Tags are considered alphanumeric and can contain lowercase Latin letters, numbers, and the underscore symbol. All tags passed to the server are converted to lowercase. - \li Tags can optionally have a version. The version number is separated from tag by a colon (:). The version follows the same - limitations as the tag. - \li Tags are matched according to versions (when requesting version 5.12, it will match 5.12.0, but not vice versa). - Also, when requesting a non-versioned tag - any version will match. - \li Tag lists in requests and packages are simplified (so qt:5.12,qt:5.12.0 will end up as qt:5.12) - \li \l {https://doc.qt.io/QtApplicationManager/manifest.html#basic-manifest} {Application manifest} - allows for any number of categories assigned to application. Deployment server currently - ignores categories in application manifest and requires manual assignment of exactly one - category to the application. This behavior can be discussed with nzamotaev@luxoft.com . - \li Tag information is parsed from package header's \b{extra} and \b{extraSigned} parts, + \li Tags can also have an optional version. The version number is separated from tag using a colon (:). + Similar to tags, the version can contain lowercase Latin letters, numbers, and the underscore symbol. + \li Tags are matched according to versions. For example, if you request for "version 5.12", this matches + with "5.12.0", but not vice versa. If you request for a non-versioned tag, any version matches your request. + \li Tag lists in requests and packages are simplified. For example, "qt:5.12,qt:5.12.0" is reduced to "qt:5.12". + \li Although the \l {https://doc.qt.io/QtApplicationManager/manifest.html#basic-manifest} {Application manifest} + allows for any number of categories to be assigned to an application, currently, the Deployment Server + requires manual assignment of only one category to the application. Categories in the application manifest + are ignored. + \li Tag information is parsed from the package header's \b{extra} and \b{extraSigned} parts, from \b{tags} array. All elements of that array are added to package’s tag list. - \li Every package has a version number. If the manifest does not contain a version field, - the version number will be assigned “0.0.0”. + \li Each package has a version number. If the manifest does not contain a version field, a default + version "0.0.0" is assigned. \endlist \section1 API diff --git a/doc/src/images/deployment-server-installation-handshake.png b/doc/src/images/deployment-server-installation-handshake.png new file mode 100644 index 0000000000000000000000000000000000000000..08cc2443dafe939d3bf5d1a77812d7b62ff7e771 GIT binary patch literal 35810 zcmeFZcT`kc_azD?r3fk@C`wjT5Cjb5AcZI>NX{8Wa?X+|3?NEU5G9EsIZF;jFo7sJ zha#xtoHO(~2;SSj{<{C`H{R>lcZ>^Ko;qirz1Lo0&bjszd0ENBBs3%h1O$g~-Mp?y zK!B=)|9Ob_!zW*h+?U`Nk+GPJ7y&_c5b4GpH2j^(;HIJs0fFl|0s`-c1O#jFk@pAz zf#XF2f-!vp0>M`V1k^TB#R|d%1cVx<%IXg4GPeZ{tSve3+_S!G$a&w=2EIW+AbejC z{%C3FaEIx>rG=Hf;C+$PJ68z8pOKHbPBZOX;$SXvT3tq-N!;4bkco$roAcu7t0YWJ zOu}~ej06?0OYD9f{w8wT#KFNvkc-RN*_qS%3a7Q5G1nyl0RgUym$@!q=71|W>|LxJ z?%e0FvSC?!A{{Hu8oDQZ&|2)ace)qND z1-X!a;kv|mk?Zfh;j6;Pr-E{JriSojrB^>=-HQ7nDD)*L+`zH~r_w{%TqB@1+k)Szk*!HN92kwsd&Zkgs!&R`kwt zu7G{ni&w@h^{**k46sXx5fBoQW8jCB2IkbavDVbdc0I4W&K%=zuJVn$^TYMd1r-?8Mm!q2(U{^# z$z&5)$T{e<|zGO%b#QfN!p;u9Ku|03=7>&vUU6=fdl7HbQmORK|g zy3-4K&G`DZ z>uwPKe&vonVOtmONB?J5FigRy7c6r5(L&B$jgpM?)n+;#DQb&K zD0#I?ta*Dz+xuITLSspBcKF03+O74z?XKr%r6-DH@*l6U=+3XL31bf*U&2_( zZwiv3u^3XASk2J2y?0i_K6~~|y8vw`ek+@JS4n=A&pRe2Bn))Aa7=sdjd?*{) zbYBB#=X)}aUSBaj%1x~i zvXAt1g`d4xXvcwFTlP79(RoUHyfMzVHB+z8b#csa@OwDdjEmDm$DQf2^$hSKH3_oO zHMeQdFAb{Rkp%zJRu%{p-+i^aq$XYb=z6fby!EM)`g3wIQPVOHiVw=Kn|jeF{(9}1 z(V-)TMw9D*E+V)&;j=s1O3ID`>o=D8;?_{ROJ*&s>s#y7^CB8mf%KDYTGo!|8P1RC^Q9=4X5AH^&P(h#*$6sG~H{NiCc;gng4MGCb3-O#W@*m-5gz7UJLa!3IlW} zyJ}iiymWZ6yD=``r14`$;ZKtB?t;uPcC`$3%VB*YVM3I#oR8Yx4)w2CpDwq4u>Y{# zJiGbvfR`Lvnidu_Ryl$;{oWkvSrq|hnR?~D=Wf5!7jauURFkJ*>Tr;pUsBD>KJSlZ zhQ6wJe3*9P&GtrOq~2=jd|eJ~`S6Y+93_W#VS|Xtjrkj*%RTn0I9Eq&%1mM^*wIA| z_vrpuKLKj>p!+i3*=ee1xLAXW8?Wu|J}M%3cYABos<`-#*H&jOJw90}aU-JMiO7zM z$)+ykQP}Q|kXI&*H#!6xg0pz!w9v-Pr>B}S6+w}F)-kxb8uq21Pr05KQ`q1~l7cG) z1lL}t{xSR5>qJIBUfq%UUbr@vim&xLYu9uw>N=UU%iK2>oi&N}3>{M2GbN|>CHj9j z6aLx5#6;Ki=Z9*oc%OQ1u7ycmwHM>M=2D+oww4l!kCO^xLyJ7yB+u%{H<57ee&Kpp zMMt&){@KZRTT1;qv8`8YC*Ce8lCwSom-?vjuHK$YRlty)Pu42PILsj2@o`{)*_HS^ z><^>l}nM7Rv-Kn)><^kAGaD=+??7Zb`dHUQ0VX z3l(1y&a9wI4!Mx14y|yPac{6H`Zb~Fr^Hy|SA$<@(}q{=&ntlam9V5l&guuA@n3t_ z?$`?WhxA)7&J|ga4IciV-EQc|>pS%PsoKVa$5p^T+o+n%YKd1is# z;vVr`=}Rw$>BdBk(M&Si90~A@J@&^49>!O^yYVzlHd^pbr&DI5S$W0-AB-mqeDz>9 z{a#c&rjB#Xi+}(Acd-Z^Z_;M*5fZREa{ghK&rbim+o#8eUxtQ;jxX7C#4e2svJ(Gh z!SqMHA@AOEe`*l2v`st6v2b^aj%4klf7bf!*(}rM1iuruZI1Hf)p8K7L{H^2|Ml%! z`4&(7Dm2=~#F#|k1{)7M0(YHF5s3vjnxmAI_IQRPv@M=G2hi9JBI2ld%DY0lX7rma z%VXV1`sU4QM>l>8`E^HnDxx_^YUY zedvER0buzQ6#W13&9SicuV3#^8;@lQegjMGdCg+a>uDoF!F+ah_J8$w`HGX&)DVr4 zsioxi6_*S1q&*lVT}}pRz|MQzVGoXpVW7a~{HNuklFvFxN%^bHuIJQkRo8aR4Ba_; zRsY&EHk>!yWp`sSKPer*O~qXUX6Dca#}QXpy|d&e`zBp|HEp zNc7$vPoN^cT7swI1$V8ltoP1wOz@+JChjQuY14}@D_8G?G*HGz*BkyZ26=5PDOYJ$ zp~}M_vy?2dhhFD51UG;~J5uZUR%LfF!D>#O!vxyl!pZv}Q|hujARN$iXG#Kd(iB&-TeWPQ58n}6?X ze_Mu*`FLA;4pLL}?eArhZ$e!?8D=@FZnaFvMUg8XGhO41%Wr@d@R{#YS|6`h$h0@rWP&nDSKX*pOu=n?IRwmo{l4;8#X1s0);HVloFJK0_lai4lug+A5bLdg`CL^B4ban zfc;R%`NXKZ;L!6kbV|A)W>5F=jni71_m$yP($pKfa!n18w7@I;O1*G1gV%;_Nxu(i z8YC5M$x1Ac15IyqFAhCniLt3_hkRY2K@o^}XvI)c7FBam%$v1gY46z%h6 z^fc-7Tu&#}r{>a~@EL)stWducLS#a5qHOe7U!U6*k2MFi0v~&HHHT(y7R1xe-``%~ zIrY3Y^A6K+SLY2wqLoK4ki&I^-p&Wu*B#-$^ z(gavE=kbQO4}>0jk=vkZY!i0VBm$Jn!80r@#w$LJ79}fI9nCn!lgtv&s%OweQ=9K^ zz1Ua}pcAE4r&T-r@@75#pGDEe1s`Ig=H z14rnp`62JUA}hM-P>|-DqLw+;`^KT^*2%J10rB;u`|AwOJ+@Xwit$n>B^f<(y-K(U z>SpktUkyle*bd9yy7i5Jx^%v$Y^{nqx5TM@<>%9_<*uoclTGDh*5ttiIc772B)j3t z|3WN`j9 z8r8>4HgQ^&Z#osEc_}YLRNVO98wvB;>bfv;=hCH1R$;nffwM!A)~+RmYq-h7w7j|E zWFjqthFp4OJ^5DnF{{g^OI_xh>s3`%CHChXug6>Wm7V1^@GGY%J?H{1D0uobjo*$7 z!iV*w*kZ&G{W;WzW=zhe*w<78FUPa{w<{Qi<7 zfzB*m<^GMzEbE01t%Y*7nRKPZKrapJ(s>tlkGSSut>t3pY2En%hm!gGw3pwcdTnpD z6+4<+zc{b*&6F?|1NSrl$hha?8`1cKR=l| zClWxwF259iy(8bUyC=y8=epSADY#TGx}8g{d%u!n{EgT4df&^JFNgG&dRbSNByw?% zq8l?0O71jj=@#0&U2fy9DRo^mn~#6A(mf(Qeo=4Dw_8$d5%1hL>EpUBRw~AHfY9p5 z55h-#t~*!m0ToXq>9aiN`WO+8OP3n0iUyB|=f6G4&8-q`g=+E3 z@kbfs-zVy8#+B+ey5fcT-B)bF+b(%-+;3iRhD?6OX|~)f{}5g|Da5??$6Ep|z7^AZ z-R#8&6|PwSh!$$!PsU&Y=t4p#uUT7~DL^9jRV#McmX$u}slrPJUw`u#PqtREqh!66 zt@m6%q#u3zL+jN(s{xO5jd+q#fMustb zfHdohdtBYtVwv*OLwd0jqQ2G+f}i+OA8$)s!*>jmy5hIWI=L>pi)DG~MURiAX4cKb z`xQ zA3J}TuZKLp9rHc)a14mK z*8#$jyjNhS@J~$tFBLy*q3iF#$NsNxMWoyhPWy<7hw@{L0{T^8Mv@)f9tv{ybidn1 z{wjlu_UYlDXCT*sf$0ib>=wU>AL;jk1i_vCuRkPak|%&Igks7j{hJ;TAeWH+|LT&^ z7>tC3#AtNX73N^XE-3d~@22-i5nn9>AMu@R+w(FviM3rv<9SUQn7VOp1tME3zw!;L z0)tJc1N^RHPxQSJU0Ii0SQJuw_Qxr95dU~1+V}I zmh`3=t{#=nAo`Oa1XJ^GE;L&#0BcBxKix+C4cnT60)c)fU&ju$~-KmdvU96RY(4ZYEd8MIM$L4p05k=<_V~V3n=A;8_U+Ggd#5) zJT~R-V%H^@erujd^dGN6EUPl&|8Ie2dPXV{NSSxu(r&fIB+<3g8=3Mvs%T zhJwdyh6if`|O_X+0?vPAWdwWQib0OmU> z>f6Ls`mWKK!(31h{Tb%veF;{cZ$BG84iGQRa zRrOxx;1}@5vTSMp@@h;^E4H`k&wSs%2Z!uHZ~fa)`E2+3IPXOeAJlt|hx&=vO zV<#mgW#ya0fQ>zP@h2U(Cfn}ZV&4(7vNG^myrNZ6ZAfc0#C`qxwRn-J_J+dXD-1DQ zN_=B%ulV*RF=;8X!WZY;56gPtzJ2@lG9to^sLu0fZ)g^vXlD=AjqVNJix6GauHzgn z-P+JwW7&3D%zn#hfDfgwrwA3D_{Svm!6Xkz zzjXb_Bu`6$Nxmz7@$_F9z)LX6e=$8^lJ5Yk_&@wsKbgnO3jZ}#v810ret#sOXKSt8H9_8Ik3o_v!Y!G1i(qJVSK zJSd|KfAlWpWGLJ4`m#phJPU1M>{Fe*tdh-zcD}aDML3>Y!ae2DeDPd15OTF3O&f;bgG9W1-lVGyJN_TZ*7A<~8_wNNBM|X|u{50RiG9$qIz+e&1}5 z$*I5E{rCv|SXVr6a{5|RQE90u%2@Q2HM}pjiPG=MlS^gSs$v()DBn&0NK~5~=PZ_4 zHJEIB9MsuZhS3~+X0~~R=8|raTXu=9e79%ERUNK2{_HTfR?!^y@yiq5T3gdgC#Uih z?n*D$h{kix0HoFZD@fF;{_1)f>2*er1%*ff`vfEscy7!}wpR?l@#^jMT(>OQ-dfk0 z5e7m{3u^1`rSDwj9exzd7ArFYoJnowW@afal}iG{QNIFp$~|#HWx#pw@Ig#kl`}xo z>yS_0GI4?0*2)$V)2UY;>q{DdKWWn*b71#tPwg5*&ukq2MEM)>yR3$U*4N`gBP&Ne zP;2Dk2g@*0?mXio-&NXf=14>@#(x;Duv*p~J*j6IJ=SK*nEBrLeMv!CkY+ENw5Z9t zXTjB%o(t=tOObtE+xVuYrk?g2)H=T<0s-JPCGo?(b2ko?5lw4Bu zClV{Y`vYjLT$2!pg@v&|2$$tMO=|L81Efz>OG?V$4RYx|Eg=4SSQo^?3^n*6v6UHC zJ9+t@-(fob26UZf%x(^A#T`Lm{6v1wK0l+2)kOM!8q{>8npbIM>S?W%PwCkPn19!3 z_j?u#C~b$^yZwLx*WcvD5K>2!LHfx_~F#M25Z7L1G>t+%FlZMM09-15wNy}@^;#Io21WQziO_|4dPAgACVlhSeod}$k>0F zrUT%t-%H=783Qrq8#{b$-e&vc8BOU7Vr zsDMmx(z*L;p5;1U34yv&YcgwvXKp7|Nq)?<+=!!OvBDTv!R=Ybc7Z`B`T3dE^-0zY$kwtj1_k za%3X+61#85yWBT8nKBUryai3PlB~TPW2uL!D7E&wbH7Y732Up^c^Wnv537!>x079Y z`$f^sH3D0|U&xVgyOz4I+NA>P=Qz?(89)L+)HW{{MwKZ;+bF&gA-lomjo)}I zy>?Sck+d58Oex2W{*91xdc{REuZ?4DODOq=6TZ6cPB(Ex#hWtIU&Xb_rtKF5IjZc1 zV_bc^x7LcQdX_Z06FOg)bKA-tn-D2TCr!J;$nQFs=Tw);Fau*G_M4n=Mb_t=0t4MI^B~#|K&1V=YH$ z131+(qM~iTp0wH@fS|nN83#DVx1);{er15ETq8smSKmMDUnv z_h(GH97ETUET}j`F}zuFIrZ@yAHKcf!XIV7wN|4#t?|;JnRmKdzkD*vfOdK7^)DyY z$AR8HcLFwsRJn)eV#OXz__Jr&2vkQrb@}-4VA151UM5-b>j&LG_~=~;a|-Nb2}=q% z=2InNd3>VtF8+*Cd1weL8cRiV!j==gAT0aM&z^zb!rvA9E|7otKw>Cq?XQhiHNB!L z8+g%f&3ud3eRs|lM^_YGFA_KjIZAL2q%C!v@p&&RigJGoSo0GTvD+MJ$vZH=o!CQn zB1!ih!>+|ZH1GovO5n)U;LlxG19Oi$3+fQoTVwwuoqhT#eMXE{ z!0x?}XR zy_=C=$EFe5YidRaMaFbF4~D7jpZyW9#e=mUU9;b#bTLd5M3+U|y3F0RwWE-j1WYSV zg<`p;7E93{uTB2-HxE0 ztyT7Nh}L2LnfZeBdMqpIN>^t6(l$HSu)us84ftessl`(shnVY5;YGOr&Ni_Jp*4}_ zc{)Ukh9Deo`$kUZ22%{5jCi-}>Bl+4G-GS_Kjs;|}+Yg}w1VXg~l zi_<3C8@YT9D&z$B)6j~d(DH14k-d+-G4}x-@|6^Alp-Scvmu;n4xBkMGzSI7X{5k- zM_WV;Iy^osL|wxYc3e_>7Vz>yHvJd{vSr4^sOugIJ0@fY236PjC=a$tg%}F*zPyuH zjOcPR_(>6-C+O;0wcZzxWxO#+lV-W7hhzX4pB(UIteYjz#DiI2A+obqd6b5Q>o~o} z+k86QdpdJx$7#SIa=fz;-r@I|SPHy;f>J5uNRaddMaf1_7ngPo=B@8$dV?Bjx{JSE zm}sGt!m@&LCh&RS&}9ltzcpb1%^RSuJOhCN=wZw<%UuNN47jXfjQ|=Ri~b61LZZGQ zhjr=fV;<|?9t43(eB)W1uAcpI@&m9ECBTcME5u2(eK3^f3r}{QKs0aQ%kvQeC8KQ3 ztUJdzsNkE3i#deGHo&RITasUh%v9`4pfH_W@3F5h+MF{ha9Wx6+BQef6Vr#Qpiaq$ zNcA1GHHe~3B8XwKEg9F92yuD`41-v`xo#Heo;#VAQ&*!JBTnO7GpfomEHoNi7Gvg|9v1MA3uq%@>2asG}B=>F{N z>?O!zx*^3JSLEWvEhv_ZrKH{_qpD8Dr${ol4ac};r0Pu;!`k4+@Tp@fvsH}vbwuU} z42;oq`F2_K{hv>!_WAI2r8&|>s|IgE(C&l;$xB}30`g{!7swfFsq6EQI!psiMWp94 z7-p4+>H+ajgq;^|u1T7!lR}d2j-2Rbu2wu#kxV4-60o2pjdufRoFR~M`?y!1^6h{H zp2)9%eWhxsF4`Q?nL>iv8zAN8$fizzQQx~@#K#KVv=(vM(Es}*+1s}ltm{VuB)&}- z{yI`{9@tRy7eIJxP1UIV*n`_%lrgxDoI(KDO+IuCM3Jr7b$I`rN2Jt4>Ic{nD>I?e z|13b`7Htj)N4d9ALA7@1>47lN-F3shzr$1KV+LWpR>Ny zovJ9-!kZ^K93kpe_N8iDB=B9n%MHuAD8--Y!$|t;hwx-3lP(#wSk~j!!h;dvZzgP6 zgBSgqF0-j+Od#WlOjXNV%53(tD6$*W_RO|>e~cR;0MQvW4^&M7!TWwpwfK?0E}un5 zf3LaG;Xxj?6y$*2Xs$&`u1LW$-jtPCmF6+(et_9<+f=)*TP^-oQmh-WgHE+P z?e{X~Vwa_EC`i=+>;1b)KD55nez@+z(w#~RiS@;ch(R6zT!!&;B?x3PsVjk{pu?-s5;VIos;lhX%)7QHhH?Y@+cJzk7{trN$wBF1J+)0>GcL& zW@l<=&6#_4B|NHY(z2b(L5gs;ZZf@oRAsVd;&gH2rE*#_d%+k-aN`(m5&-uCaCJP) z-rlXCM^HnfQ#JOrr3tmCs?{%q<+0ARhO3zY(u{MozhE9S=o8xGvsBuZPGRG5?0r&z z%b*zb`y)H*uy+VY)8!g9euDt?tBCS1{94HP5wTwRd<%{6CWVH6!Rp$#JGn9T0_wfl zJ;FN_NL_fCFy1a$K#gWDiW2#2A!g}Ji4WuVnBBAv)*Dh7NT6CAc&Pmm$Orl4e^7qI ztF%D0l$24kfBVljzaS#bK!z&B>i>NDe{Y3M#Q(u_B~OH1Sa5Iysjnqu;{puni(MI; ztUHVU77$`OK@Tx$gf<}h`SYhmZ>h`6SFcQ5!5Yvd>EF6_E8p<@@q#E&4Fh^)`yCR`Mr@@DVrF*`S?DVVs@m%W*y^M5^1^^N z@uj52YQ%ka4G0Ku96uProLXK)10mrM;NZ-0j721n#)%UK2)3tdwL|25|y5Id2}T!ZRk3)GfB3Q#BoHgZZz%6PRpLuUDlghu8ZuPEJ4k;|ZF8y&K>*-^xF z7aAO$0v@PkJtRQM>VvPoiR{!)qH5bZiI4D#yYvAe(Faic8}r>cytkEiu7X@LBn!1y z%|QH~03M`MM*YFw#)Vnng%E0SChhNyU@}U;F*s4S{qYGqn>-kCL$n9k|9KvHH776w z_=h|@M)2pyJw&Vde}~2NzkRl5JJquHBg+bq?(8MQH0i-BlUbkJUec*Gs!r(M%PH_0 z)g@ITTi@}}5A$I;O_s^h#R;(F)HnT8^~o5Q@_A$gf5yI?`rctqmO*Fs#?o^$ zC4QJmkW4es?VL*o!bnX2O6V2shKWas{<|f2u zQM|h9QLK`wi_cqzyfshk!T!O{B}k3yJ;7rLFW+HCVUI%MDF3_#EKlLQe5!?rGoG~B z6OgZ=lI1s6o}Xw~@;*omYO1$h=CQ>0Dw{`j)MO;PWpY(p8PEj8%(u7m-dUv2cifgR z*V#HjtLc?y*;P=ddDa=h!Zn_K=~}1N{jkg{JUy1Cn~)!H)#9B;$)bVVRM7e)W9wRH z5l=>LmmzJ}nuqzsr-00^_l|PvkLk0FwV-!ndCRo+d3RUFKGUU{8d{4bJ-r)-+cu2@ zEWsfI>zr3@DtI;$BCS5Gtdxp}0sX5H#2_-^$9!ZfRc=K`faAyk*yf+!hMO>hWHhb9 ztDxSdaxC7r$AU7#L!Y{p#;o>5P;R2)bav#+B_}ca!ON|d?#;{TmXGT4-P=*!A!Ybt zl_^7Jn(ZoTrYiL>g*9zi+E>i=28g;RNgaN<$F({hJ(3p#@TZ=00UNI8s@If`DMBI8 zj)_{3bPcbd?xKg)T8f(MGF{@k%f*TkoNmXzNVmA{qw*wk0A=TO1EgEehA zFrS4M9%#R@QjM27M$MY3Q_?2!?8F@q#K$4(1B5sAZ47H{u>q3kNPA2i9#W~JC{m}o ze7A7D0iWiVWsN_vdDeL>5gr!Sxm%s+8nb&rW}7la4V3LNS*M=5b)B6T>Q1w4PBXM~ zGap!=t0C)7>=r1`{~+zrZEUsPK}BHxUULwbh0d6LC6-@berqck;<3UzKCRF14%fN2 zIYe*hMmCW>@6Ozy+v%vExTx3gt!}006yJGQ*3!q(6;1KdlgoWxh_E%HiMLL0{6ZwZ z?K{h!!ba$iGsL+q=fm%!MV(@YgC*l}m9U7*eWD+yT}^C{V_&^`b-`R6yf}u`y~xqu zBhNAAyW^C6_EY-d?`~EvTiU+zx_sJyfxmZ*g>FdJMRxJ~WT8}I$?{?ik8S3nN2b?g z-&ilZhxpk~%YBKF=QP4YuS~vY8}7i?j<&I8$9t3w>U4*dHIIdvPb|^4%^X}#@v8ZH zPCc(fJWbo;xmLm5_S4-TtRHv@S(9*8Y>SdwANOOv3dNR1>qJ-H_k}!Aa+H8jauV9{ zK7d+L9f%b?u>-Wbq=9Voc!o#90+rM*c5y5Tn|6cz}6D3Nn%} zfudXY`D)TCigz_7k5^S1H*Ba@+&WIvX~`mNHkO=x<$m+^@sSb=Ip+FA)uIa3DcuR_ zc$1&Brc+}+LmJ(QozHXibhd0o`vM;DcvQ~SEm&H)#BrU_jaqt|mull4s5Un`bgRsL zC>5VC$5%c|>%8TbKo~v$Q`{;m+ZQ)Ln$hQQ#gR^D_RPBeFLutXy6nrUr^&yf0NVZl ztzcbc+gonqU3sY`r)I*`bZynBsHu%Xl5pf#YTW3+8Az&RaW}LmFigLR%6iwZ_&f`5 z9gSs9bG6y~=F{J8uLg%qp?(53U+6#@uPvNq7aZBH(Oi9=V9RotjxRk*;{EcfM2JeM z+Vtu0>9txk-gh-2OQ|kLO;?Mrmn}Tm$XI!D*l1Rs-mI$Ge9E!JRAIQ@bS1|^VCHOzuCckAt4Os zqE$kT#=Ha_Y!!Z=-ag@Bjo^QCT)9M<@0%wGn8)a*5C z%!Yj<=13+3w8ms^u{oHpVlQxOw9vtn@#xe(IxYqsSHvcCif0Ht<*1*ew;cI7RG}C* z{P~|us`ZvwODnf?C$>S>fM*@drQRK3a zQVq0d<^GXD>;&Z&0P&;)VkrON*dHR|9{$DSmI&(qCy8R^fyC?o9m7(k=!yXN_hB6W zAGIDbM5a_2v1$mhW^cY_rUywY#{tO7cgcy(|2o-PPfh(Zn~} z-X(T}Qrj`q+BR4bV2l&!1{>of5sDh$4*KlzHq$Cjonj!LMTUF~1GxK~`#cf^>#nX_ z6x8Rn>zaS^B`W|bdv#m~*ao5Yxxw$7;|mBWm!L2P0=Sa3tD6f-1)I^%y_e9x*`!}y z$bKPrwKIa-^e&)ZIGy_gk9dvi=n!EaqFO{eqXsCk(*U~L3b97dnn{8>Fb%}vPV=>V zCB^?W@^%BDX$5ecWA{SPm@A0a@OEwEJ-MbYgq(p@n}kv40rULBiw@zpcdQ8eM@y4W zzGxPDzczx0*JbXWvoi3@2zu`H<5E=zz<1^dk&ZOn;n-fa+)S?l!f*`L_(E3j8@e^e zsP^(8k9ny2;A2{=EJU#vdh~6D`i*Jm*O)?S6#Vqs?g9DZR0qV|KKd(wf>XP5=@hqk zLL)*!`(U`96}(Me2(x5ytTRYEk@C$ssw)Fn;&tc+bpm>%&;aSpaMmFSu6lOTfb!gJ zq9mQriP}30pk%7} zJ8pSv6p(7S)b>wDgnl4U&=>=d(rzN#8}m4w%}f&S#W2t(B3&4?d9reH%hqZb1ZM66 zi60_z&9H6LqC+(kzQ3D5aKgMM))G{{u7W$I0fCMzDBW7}a|)R;d4ESLqo#l?eZRVK=Auv^-I zU@&id_o()b=yq(+5U>-4-Wrt|{)&qx#W;Nb(uG27JS^(Vd|{7AfYLK5Wc+J1YXxk z;1lpBj*jL;>MpfXnGNA6$$$yh8JhN(+6F)5P}D3HT{!XKySC>%2z@t8 z%$$zIhYGLsKj4fv=*YU`)L%ikz0}id;-K9g=Wq!BIg$1h?iZB>qDQ4qi6klCn3JYX z^MXEGtMaHZVaTu30gzk*dgOH=C`6_w0qM;GO+t}^RX`M%ZYh7FxW-?>A(b>ZSao-- zi25jvsj2A}_u`)~m%Q<>Ee*l-jcu^tz@-;^II% zV|87VxsZllRflF$#p**sVI*jzfugF#6DNh-3%EY4T*_SQB z(Z|l1hiVYw6tzp=h0J?xj_X-T30u==3XoQpk5AADSl{D7DR2Tm?z+6)!)n|=Cyl{u zbDg7NCmbP&_NfrS1i#e`KxxN&u{vw&x%S^w@b^sO5sC4+7cggbjuhSgmfQCO%0Z?z zk^A|NxRx75y(avRD+FjT)Vf6udJo+lgtd97G4if4UZy-^FC`#Uu%3CvX)*JBz1WQ^da=u z!Z#h8g2+*<9+1TYjrsn*Z2U$W-q>iAiYZdS^@hzPP%9 z&Y)Zn`n?l}@p*5i2|Vl0HT^Y2?Kv7Bz9Bh@FKMrpOFQ4l&C^xO*JmO1>;B4s?x<(Dy)>ROl%xv;E#4eQH0L|*fj6*L&~(>R z#3T7(`Ljsgc!YU|Etz5?U6t!;ea{5oF1`8*1k{2;lWQ} zsSS!&O}9+&RLM>$n;`UXm={hxD&i>LJZ9<<&Z*P0JfmCu#<+1eZ=KhE}gZruwh z2l7o+Lk2$i8tc}&5&56#58yR-y>YBIuP(_xgjE1h=(18kVu-rPZi9nrFdgCwr^n9# zzTO$e_|*_Sw@DtQ*wu-g`Y!0M?e~#HKO->l8|Oe?$(Gpw}D60U2yix^w7mNW%~yQ$ehIbK`TEwBn4H; zI)2tQ$#1D$Tyo)J$dAe;F_!AwSlSm<8!4;=Y8N5G8Y7~H+9-i$oyeGT_hBKsG>oGM z88*{hUP%Vf3^HWBGjDT6$H6fyg(bWm-U#99MqC3D|I5+o{nm!VB4b*TrW#lYUC*`H zSl`qU(`T+t@1mf0&(XOFVS0+n=IewKB%{aoJ-!g;6X&T~k+#)P+K!w+Q2y5JlWAh~ z5cEG#c~QZk6u~aJoo*H)adlr*2Ja(z>wVO0WvZZ^SsQcgVn4;SLOV&v!h+B-bPZqc z{W#LH+%%@!gsDCmz;1OT$wEW&Xg3um)kIli{rpPv1JQG`Q6myB7754$o*8kvzR&fk zKn|B+IlyM46q`}1HJRF&3~eIC3VlEy7so1AvEOWefI&ovDeJ&-_n6<=YBc#UbX~Ra zSJq+$MS$O^Ikc;BR<|rTK#0B=t;O6xlis@THaAHI84V3b+X1qGezty*)!#2jn;*I_ z*M65{ytsg&I+mxyJQ#6Jj*++6X>u%~sriA%!X5Frp@`wNk+`5GultS6Ul$*WA6pC) zUT7ym4}he?ejGYq-jTq@Wt3XPYRFZ#98klJxyg(#ZingjNh`GE+tVTSN;+~LLhSG{ zB1X4~>@AUFR4J_V13)7?knNv^=0cUqS22$hnQk-Zk-T!Wk6}QkW7OmFX}+l_WF=D_ zIz}I-FJ!+8%s3vbqtG&Q1cz!TImXs8(V2ti(h^BogUZ&f#AHEvc(Lwn7|C5yEgD{n z5z*_G$~f1arQL248@{FUCgI|3&qG7+Nrg9vE4Ca7xP9QA2~pqo11cUat9fvahnLiA z4`>Ur(!hRGkfa|h#gQHFie)N-Ft%l|H5=qrNXFR9t1RPtnaf;NFV-aCOw zYLtwm6e7U(N|2&#W)lyUFngg2r!DX}cqg2hKOg=?QivM*$LoT1_HVxe=3XEjFcL1f zkbCGLDHnP96atM=a+*Qa6zno|Q-n*TL9eG)yXQ1F$^I}a`OtSv>dd23j0(jMtU~8e z%INH37ug$=jY%0zyxh%?=X~cQ#KcYMdT(mAIVA^z&um_0b1sR!VR}ZYvcY%%pm!>6 zjr<*joyaW(u`IhRl@{LD$!`{^_vh&;q0GFF$-Grdl%;PctMOX0tHeueiNyw^)-*=@ z%F9Ld>s$*_CYJT%5AL)XsPA-K5SKyVBzzV&6dS8n`4XjquI&Bg?-=*m$0$I`Tv>7c zsJRpAG4v~`a{hsbls$9M*EFJwtLWmmw_25|mTF+IJrx5Kb~idq>V>>cF@-{0VsVH} zWv)*#uAj$WKTkV$06)Q+2rPQZ0!_RN>!a)KzwsN%3x{-keH;f3 z$0cqa1KizW1TR0bYwrlL7C;cx!+Bk8e7^|VfX3yAwu91}CYI0xY2NH7Zp0NAf0Gqx zAwl8Z<)Y(#SZ&$^?4;rwZl>5h9mA~ z@91WJ+%UZWrx7?;7DhR6=;FGD{dt5Jdo#Z^)7*)Q`zgBfdo(NR3s z%9BYvmeK@;LX-C!=OG6pVTed(au{`^K{Yqfs+W6wr_4rv#D*f6&cT@Ge@O3#;b3j| z7pc?#h3T9{Fdd)a+`!HTK!b%N`QCdcL!WL58nu0tM zf8WEjLlR-fxXm3jGC^2nB_d{D_Hbf{SVVsOW(68G@WKF2F|H1fzB~Z;*SOAr6)Gwz z1ypp(eWWG2CgrV=7(8}_<^kr(!I>PryDfwQ6wGymcMhdX{UvTWl&o@I9*huAePgmm zgTfjyBU6U9vK&${mWU+%@imY*G=d;bHeRCs2?eVPq34oxSa}1B@n<-sX~5Q-Z!E3t zau*$ghXkT`+R1%$;rzHGlUwIi$4gK0I@mLr(tLUMTc5~>vm!~U;fVkt>eP8KtD8>J z83Fz(zAkzIDiwH1;*4<`{6ehi2*3~-DC?1IJ}bqq7oLG#+Qz?lz1}jerMve4e$=Hh zv)45O{P`{Zm1dkXrZJ(ozmsjig( z`F$-~Oli;H0H#-CT8eR-I*4y;ybRsJO5Vz;!Y&=VM<8I(uT+?0qo6zc8_5hy+4IFC zL!Bev^~V(XP;c4c{YRE$WH}5}1EQ4G{}!>pE*~ef+tD)un0hf);x7V9>`Z9XO1#YQ zeiO`^0<}@Zd%8H}DJHf7-jq5bO_>aO(v(*P1a^1G^`{tD*J2z9`|{nEr|7se#`AR6 z72wns4W4T|N5G(u>Z=BzE@dN94>>w60K0`VPAc`%d&R)Lh%!c z)AANrsNyssm*Rk=`T6tb7SI-JB>?T7&>ZmVrIMnezXt;waUdLDK~xo{xwi&b3)Fu! zao!KXnw7QKQ?2ye`=Jq&Rpo(0z{l5*#(JOe`3~DQ_4gaRhXM$$~?{+&sw2Up7@duATCjCSgaT;H%6x*m1(Bwfdh^ z!n`pvL{_D<63u!M%ApBw4W&j%Vd6)YDLAe(sR6uXAAe^bGD%IS-lQCwZzlFPom<5M zVBsF{t7Di-iK?F-hFzPQw_u2L*fU*5?-~6-W7RCy^Iqn%vF}WwyXF(jH`06e?uBmL z5&-IxX+{*lcHiCtpM=D?hk+Q~^2A>cW)piea?BsrwKg9mx^@#0FA!SF#EP!$&$U&C zQ;1HnS4@uvPtwq+)}U55V~kV-6sMSEgD|#4KC_ht{F4Lfe_YaO4Or&WMb1UM|F|e` z#F~(fM(lHHFqPl6JA?4XERY1JP~bf3!z8^cppBW5P_6Rp9~lEV11h^mn~H({#@@z8 zKz}tj520ysc!C)47P7K0KY!`}c?Jn`(j?M;jBPl|R3(tR_TY~i;(9*#lMsBlHvO(; z!m@Z1!Op-0D?fk}5-~{pVO@2Qe-U>0gIrT74uHjtNVt zeRb(mg4`Rc)>JjFh1p8sSU8Gh2@+KFJXH%c@|`*ZFF@?LAHr-F>Nh{N>a)#H3LyO5 zXQiyhT;j@T_K0Fwew?D~0hkN8w-^qPs=;vXQQU+$0^E*fw4fsbTrUCqSu8^39>^fl z6~SvcRyGAVy08)xataoR4AVi!{QZc)Z}JjAoJm9XzpHQqMI9m+pC2=6Nean>gDF}E zcnY8kWvAy0VE#!sD)hsbr_|lha;;Bu=jwlJ@2jJt`rdZ|Ns&hBPC-IYx*J4FYG{<9 zyBo#;q@+6qq>+$LMd|p^UDAy-3~~3UpYQK`|GMk0`|oufXR+4IoZ07`z2Cj}^E~hK zE{F5(bkYPIyH_Jthmit5F%h{UbHMAy{`#*G>aYEFkZc2#J?3K};r=UZAcvV?optw| zyJZK=7gHdy$M)*{7bj*jpbKWlL;$CH+I9zTa`Ee%xoxl?u?C=H?}-w~zYz1;t)AEc z5C9Jq9H=-a0dSbMeM7--?+WumAd`UcvR}CbQr_C`SeC}+j{kwI>xKNUPjpBW5FR+6 zUU{GHSlwsW{sKb%1R_oVH1Ng+fL_5cY|gs2(p#|n2cDdE2+vg#HH6F)k=1*9(XIG4IvGQR--)`qqV02m?^74^}x zvt_EVLjVO>P8hgEoNNrWH@5%}gDLl0pe)~sdB#}*)YSD7`mhEReY9Ed04ouKXvAho z7qG~)T_*^!@aJWOtHD<|(BK;ikIA96lfNIoLCCG(k(7eN2%+x)xiC!J5FG&F$u||+jMT7iWrLQ0h>ng0G}# z6{vkJ{ReXoV2)`n_GKhfL0pRUiu z$vF@c8+$#G`y>Vc7<2Z`C)y(s(Fj?+6=QmurD~)O>Niy|$`7`A^}+*)IBq3p8?m8YeUs!yp45U&xz`*}gEB;I?o^`B_9H|dnJ{|TwG#~ih+ zHQ)H>SA&B@A=$iF)8!NW4ggJVoL&O}>-vnuY;Mpd2gNPVnpGkbmRI+s-nZUPZ|cw2&Q$O1Gfk!1Z#}(q*a|^AY3R>!@_ww5S!J}sM72` zoUA*AFM6aqF1CaiXI05&w2oVLlaT!JIA*1DPO}I!LWMn8&_<%9;l87db`(eeT=8LB zH4WbC;%Uu4+7)vTc?rAnS0xgV|HY4yXmki>Dv1X*vX^M2U3e|;M8!cKwZu7M@eUSR zKVyix_Rm}&@sq?GSU58O&uj(8e5c@_G4Fe@n);W#iy;D`J*H|+857kY@MJn+D zGT@D0rPZ)xsa5jqJjkgwd%<_p>Hf(<+)`it0pR`ZfxZSy!BAFEyfbOlqt@OuB1tQ! z{N)aCKZ_PBmaCs+JcEz&&(yZYo;14b&X1igpK@`ueoxhdZvW0Z*3r+Ort#56 zy^0H^(%v+2kFBlA zZ%S4Y%fv+_+W|U~%P_ltcH;Bmjq^O+iQE*4{`|U!w>=ep?#YS(238I0*u2^;T`_*5 zMx6AK=qDp7qGzG(nfv}bVE5#E0M<6YL3dx6L#^LSzje3rXAfL!C=$}P(KELah5Pn| zj>*D4JS4h8cMJ-75b|N87`rNLB!5-%L~i;H`%V0-UJ90)N}H|uZ8-5v$0>BDS}?%R zj!1jKRR7H3OG|Z-?it&<{#M57c+FKsjn}RB0?SbM=}(%RUCaf`qjZonrxp9xckHj#?>A15x$Rf5ya=8F8nSXk9(u2Pe{BO+#BO=+c#CFC#OZv{ z`fh}qYQy!B^|Qzb3#b|pf#Tj}kEaimAjQ6_4k&&$_zzRbeDkRP)UR|==gG$LhO5Wp zHBuJF(;XzAMt>Chwvx=~Y`lE$)O`vsdv|(8bJp0pCpLU6dN#3OF+RIaOnpkF|5)rt z8N@Z^nTJSghnBl{b<62h3VgRxsCH(SY60%Q?_L9BInzCJV9MRn->pq|sl;MCIn1nf zsFi!qVK)dfJUxm`K9$)rtg%`UdXdr%t#+u&VRx@eX(9g5Ox8HJf9_Ole|jCcu(Guk z&~@UjvFcKymn`i&al9*elBuyO(pWIARvfRkP_r|$D16H9GppmFVwDlh_@q2wta1=Ku;^nK$ zZ1ptsXkFWVI2hHN8)-*Oz zlcU!M+L05tW_#}92N%^pj353b>{+3(J7k-CXGtkTZD&4Qe9QxTHpemPKPhaLYOiY^ z*|l()*gCvQZ89C}9jyswcy74@Z}usH&*l4RetCY!P$_SGzn{0*@Mmwv zK>0f>7X$B(QXFa*IP@kTwtsfC!A!J!a46r{v@%E5SYoqU<+sfYF;H$gDHNO>D0O=G zS#*={Rj<#U#}(E?(Vf>R0eHFThSxk4j$IrrS@j7HYBm{g!~L<(HM7Hu>c#lRQ=c6C zXKL~toLXDqms@>>vkT{1!ZpK%HH>x{ezm(V&Jy=vEm_SF4gKPkVrjX~yfjm*O<`Xe z3GI53-KB^Om)$=0@vcpS=(u_I$!?)(zSr1t^QJ|$HXhF!0z`y{YQj!) zs1%iT`G!2LRsGX^{nzES=L>jMxSsa?3XN7)4it5ZsK(DRO5{(%c2yVnZ;t!->n@Fq z;QvT^jbfxT_Dbnp8oU3xQsJ|O1i(M+3*5g}{g`Fftw9wg35>4=F$*-oikpCCagIe( zI2%TlFgYDjvh}$Lhz{*0IK@UHuGq4~@;g>fuTQPODjC_VF_h(;V>&^60Il!2Z@|0s zYP*Jqv^wh6U4dwwQzv1A6+f(6bww8>5?(L$$=*@Ns#x6-4-i|CD}xX2Z^#up-K{yuoe`^ zeh-!{Vtoly`e7*H^m(*YThc1;3)}14yY%*G)wE$<jPLD?sUHdQ zxUD;^+PHOI=nfJ1gbr&*E;pDcXXBV(t@Yc35VwKIkK44|*dAIHC+!LY)4 z%1N#_vAa=F#ADoEF)~PUjzmmDZZhi)@!m1?!?vmhVOGN8+Xos zs-q~gY^pbvq@gObsbZ44_*TY+=fuY-wfSofI8->(n=YiCYql;A>`oJ9Zm&Mef%oJo z^}rgxM2Wa+@qFn0h*2Qn5|tpdCAEcCsYr@qjB)6(y;`2-cx~S#F^>s9{qB5?n@{1a zIEki2D5#1yK>M!g?6t*;1651x#LVY^;ERFMi4iMh3cf4>VE~UAaa8uB zy)Wt+HdVT^d9^(Ok++74Pg}EQ+Z!EpkgNT8@=xd?!q}By)3kGXeb_97J2j4>2-74$4H~K3V8>!YfTQ!M&Ry_pU&cV&s2PD zzog=VY+>8$T0t&?#dO(P#dwSf#dp+&E0ZFnqUDKv!zi;YgyJy?<=r^j5Q` zctEM@9`AMt-E~3V;h0nT?67U8eNMl|YL<$_P0`I@6W%;8#l`)` zb&GyNoscA< zU|oKzTd2Ko5$)$Xg@@pm~j#4Q5^723|-w^nf|RZL*q(r4s|=EL#t-oGJSr8 zv#2;nSqppbG_T0Ju5}IU&2!f7LGEBh1qKiWmU)JNPW15IcY@e;Q>yKF;St>+0~GWN z^t4%=9!Q6bXdj7GhMB-%$kj`V@$B2uNGNf9?Ub}^2~MMo>=#p?BloWO#}k{vO$`2f zjAy%~MI(KtG-*%;L4xj}IF#566T@xevUKZ5+pblm^CLtQ`}5;E9jsHBX;v;|ac|tz zJHJ{5`MR91g{^*IyR_|t@m$@Bvnf6v2+F3`4IO?>p&kF(H14SLOf5jMD#aeifcgOU_(*&ZR!R{ak5ZznQD+!^BUVe)IiYXz zujqMufx)CewGtj@O3umr&q~oKcsj9lmG6>?I=6A+n>5D~m2j7s@d&ayrld_E9N_NQ zVHtdM?Z)HJ{-D>xh&E|WUUPm1hT;5Nz2HQjWovg1_`C~Z%Lycj=a;hR_FszkZH!UgoBN18qZL41!B*^Hp; zNikPxrx*wL_jMb)cs(+lvSInur|;|K&W@JiW~#Yfj;{AiaG%1DggHO<#6jF6Z!TF? zdyyb&>MuQsRwdRqx#HF5?&WqiGkj+0E~ z;)9s%R-Qx;i~7ozIv0bigu*TLjIq_%A-p*QV|g8ijQEqN=fZyGR}`#%T*u!!avOPl zM5^94HYlH2g%Ju5- zy>wWeIV$t+Dx1k`R5O^_V0QJAy4jq`8Il2 zrHTx>MIp*mK)9dvjs+P3w5i=u|E~Aykz0y-XZ_{t>nDQ%YsMKogzkdJee)birKmQ| zJnFl{arC!4@k0HR9~K;RRCrI9YpI}V>E=7_d|fI<&ME2-q8+;U`c!ypQgKd_(#?-r zTrSAhoNwwV`?_Us9+x-<($xw~B({kU#92uG?nsEfgoQ%J))1ag{fQ)_dmL&X+!B<6 zZC^S#50pNglbm zo&hs*DnVAul=+>Gy%;2{C?u@O6o6t+dY?~v{hd61PazHjm}@Ka4%z0yj3%N0EH1U^ zAy*7URRBlZePpGl2GL9IgyqP{M(5KeQIoQNzS>F$cA);eaUul&eJWK#5L;qhA^cdL zNk}7#-lW{cEWc+dT4W+{CS~qA1r&D%iHNE>x$tYI-|v+3f}h+K@!kLoF03kG!nQQ5b775OFpCS52z2_Wb`%`aeFn|F^q@T9^Q$ z#JGtXeo<~T5>_3($?6#nfrVVUVf30iAuX&2^T4Blhiz64#R4;J+hf|f^Fq1GfF7Cf z;0f{-QpPs+-sUMp9MKa-(P5y7fC9)%Ff%t-0&t_9E6h%a05fiZ!FtF1tRJ%IaANA%8h?(2x)Yi}vAZ{@!DMg9DBHfZI zj6WZ|Lluz;RK-2U*VTT8GEbYe#_E|vR`Wa1R4ZlW=E?}U@34A#c{u~?u4e!_b`3a) zbhGXBkIRV;WWmzSh{VnICI@%G2BpK+ae}EX_;*amVncz^;}nFx`MiS{_Y3nxhI;a5 zo5&AJIfX}f@DQb#OYx}irkwZhaFsyWBf>P-#yK;sV8l8n2Vnj|HGydWBDe&iUL^;v zLZ0FmS$v<$6z37CO~IG=KBZJBmo-aXjO9Z83NUmC_^Wzkzfd|3E<-N9kz^8RhuT0MwEM>vI~q3<5;Q zs*|Tx%SGNBD!A5w@lb~gOx##Ri2mnkf`e2zO5@A=p@TvTfFdLHZGu_=tv{)4?R7(I=rBC{Phnr(T@z$!2Y8yffp(vemGW*rBl)ZIWcwF24^1FR?Hjetv?q5Eu|6CpJ< zr(+Gs181S}KhH)45N+**N5(A%fN5bxIJ)J2`qTpqtiHTzGWm*PxCU&YN8Lm~^#kX_ z6-A)TQ_Te$d>;25-G}V#L^?Cg{h_-cnneZj!qn!s+~mHw5tid2dcvUYg@!6%2a2hd zK;=FKKpwAt4C9R1*|DAb0kqH7oh0la0c2R^1h75FVogButqatKtiY)PWKcvZ0N*Cp z-Dxe5Ymnbd%9ZJVS@^|a!3fkj-~icxaey3q!R=K53?5(e;Zh0ZrdtZk0MW%EFx}f$ z@K98I-@BWgoh^6biBI78(`?y}ms0rW(c}H6stkqe%c+HXCkDZQXL;0h=^7Z5lLk&t|5k6tGlV&?XZy z72ZcoOtQF?0A_4lPv7sZ&~^uvwfYcg6sdra=RL4#QgC*5UIFQM zLn$&djwtNwB)3zRMA{XD9*}uM6?pN0xr1RCCFTEg5F9DP$ z*$O!V2{y&)W-?ho*+t9Md>oTu;t5PKx3QIuyT?wIC>G^@$0mBP;%UN6eI6Y+hPV&b zA7SdozP?QopYIi-RoY&id=7{&FX#37*AZ2njbn5#;ePV#`q>a*o{JE;;5jgYl&^@3Swu>p7V8SQsm2m!@VYW4;eA z-jz%L(j6ySLi8s?t>3mw=lr=Kaxs;K!Lgp#()Rkfmv?C3l7O_n zZ%a_vQ)TkEu_B%aCJ?~cxO?lA8F;m59SVTEmr9hi?3;0?E9A)Q^|C3lWSe!TAZ^w! zMh+$<2-Ep4`R5Z3;o%^89bCL8$MPuA|VMKc0KO+ok%*8NV>P{jkxQzxVRR zg65=7Y&_=)TmjxycDOV&k!z~^iVUCM;Kt`r+?iH9k9kbVR`)e=!UF{r#RON4>lWRU z0L5tv4Fhe>9PiIIo{c|=is8ZjDW(1j;fP(U`e1xaKz=imEcD&loWUtKSi%2qD{`3H zNTo4{9O^`Ted%o(<$_VwBSeaa~-Vs=cIrg!7Fr>s2I&9WI$W9#0tBScz;`3%tT&+a3 zUg~sxUr3crWXBit{OH~O#lhm^zf!EA$yJ;(<^ot62?_gV7&Vb3M z8uHH_6i9Q@Uk2jnM7o4E><+%XCr_ZlTjKQn-LPoGEd*LJSezj2eP}lwP9QI1d?v6EMup}xMIwmqmRvP$cLo-Cp?R?(vM?<^PZcrkd1gYxRy*9 zsywjhN+HVZ^$E@#!95srVleak39xayho+JQm0SV&ec}So#DBU=jmW!Ik8t`z<`vT! zaM~%bdu~+)SKj4a@bGL>lT9CZ#E_0_p1`ww{ja1i;6viitRU}a$2a^pDR?zcSL|Y` zanKb)-u5)SFy6%Ub?;Vm@yIjoi>UlHO5TjaL!(>(Jhkh=sEEW11?d%aN-UA)d zQPem`)6h-$Jn6^IRoI9eH7pUyQ@Bawtauz>w4bNsUnLo{{Y86SM5m?aN=9$2S(o{# zkJK|)T8gg~0b&O%Znh0A_$dIhpyU!NfiymQn>c+KaeYuM8=Ex%X>g?0l=To;@k-8qqV?Q~ln(gt)i%)Cgdta5s z&&JpaYSbBD{+MU|P~)qgmCvW|YMJ&#$L+^kzESV>snQ3EpV4*L^L=W^*%nGD=k;Uv z?i<{@s<^NjX8c%Rq8@`Ou;o~3@gzVi@&RR3TvMi7H5g&Ng)v0Vu+XHOd6SdVo4N+iN@3G5F&$g86v88*(pB9y^N6uyP1c&Kn=qU;|WA@t+#gMfC2T_zLN#G`7LY zcZRMxro_pA3{0sK7ZsJNE1q>b5OqF#e;M3}Hl_60FVA_4j^t~dND-+tQKZ;U5g&fi z5-8(81?`3Y^}8%6ID%;}DSfndxV?(TP{syn{Y+7r6GvM$4xhO&u`y1MtMGdM{FW&9 zAMk<-#Xngq$IVRsIald{Ngnap>h9>wk^sZjJ!zTqeR<=tUQx|2ykd_OTl%hQ$1xa$ zu=95t&KEYX*AjotVDyB?&8)=V!DF{*DsvW-7jkF#De7)xoLuJ@HBRLb(jcQZ!Lgs3 z6J?dH+V>Pi@cYl4qt$t;iJog=hp6_R&At1)3RDw{xc9Hvxb`#Ys?KhVs3!%LpF7W% zrY|vUiReJD^e4U?bR8WwhkB>(Cgdu5yM3#H@o=m@sCTiUDSbhm{6t+iPHw6m&RmSS ztc18O0mxvV;9+dIX-*C z!4jNrZX~L1Tu$+v*SQkeYjm^)bYw;uq_=v-H5spO()o*c{8oZvjSGa461!x#2Fq!x zw7U=bNw$|G?0(PKf0yq?nCDRKh+{wrG;P1@(Dc3udGbv~2s*@l7w>)RbFQ0DMh;X% z5jfBP*`6rcA;gY*rKh1?GC>zFLHq#S;vTxi)%wo<$swL~>f`9W07!A3&2_JqOwF5l zohj_dcYc{#d*}4h9lQDs!l65U!x~DkVF1_eCQ~r2jW07B|9&E7&D3^c9b-C zpDe7JIJd~Z!5WnPc*rAg^PnDZEyse)#kQ>6GFt?u!FD1i#uz%v3Oj}L$K@*_e3yQn z-vsKLzT!9g!a_f_<8O*-`ATt7*f9`=%tb(+?ufnK`3(w01*T(aj^a!fuCH%wkB=N; z&&J#exfF6@aX2S5RO4;RtY5As^DUkWQP3ATeqR)z-wGGkg$-l9B0CIsPm(`PlW z6sJv!t{7;_^y!IXP!$!Wt9h&XfG0ntnm}0)T__I5gOuG)y&J81Zz!9KY^bl#PhO7A zf_%oi>pYB%!4F7p`?NK+vvi*2UeP15NAZ!Gr+ui1^ia@_>cJa+zbq?%T*jr93m3Y- z`Sc!qb~C@Y<`es7x0Vv~>#(4|tjg<0jEmL+hj~gJrb%98hZK`4@$UG;grraYDSx%~ zvM6#mrJW}%a?>NgGrMJqMlO2&@7|LHF}m&$VhMCj6m>oR*u*WvP2zTy*=NL>K{T&`>Qgo^+f|DO+Pf63BwNDp? zl$gqiYd8ly=1V{EuPYbvo_oB-(~LLpb9+~(D@H}US0Yb5NVLJCWVZUm*~|1+Ix?JW zEu^X{h#Z$SO;cIwEjlSW29=Pba*}jt{nn(;mQ*fm82Z6VN;O_Uvh;G!-_3R9|+uDZfB7?3*O5J9tQvY&C*xAC8*%ZZ%{35{fLHK6;19hclE? z9#TllA!AVNCqCWy%#ugO6QM=6I>i^Riq44K<&2zye&@L=`w6|_U%SwCi&%=rAZf1u*=ree^o2Ht~* zqscTuT}Vu0Ay^R;L>JuMc-pGziP6_6vD=~P)KVxBe9V{E^>Y@75nD9OquKjOX1a#( zyPMbKqFH$Q=@&zg;QY1QqT2ecbqGchA1xam7Umu+}9McmD#=S2fK^~xw!Q|sKa6} z>OkN4;i^%F{6f~N^gQN->s)GdlxCZmt^pzJI~v1uIVfo((eH_>jV|D^#f)N46fH6& zh{*_S*7LEyJ-m?;90_qX%C3Zx1|9MjmfY`3yxM7jkIB?WbZ~0%=x)OvZ{iW`1;4PL z7jkZ3TP$HPo7J6ro0Do2jlr4mx0JWRjD?!6T`JL8m2V`%!igP%=y>e9 z6vOR0ggH6yN%sgX)3F${u}n`DZL#z;T4VDNE04cD5-ICeuFs!ZuGE2ENVpRTbV*8n zjls!7Lc%&wl#$f@>P^plyz+kOxtlw_a>m;w11EJJpM_>`-(O#(mHpppz8YwOADx{GA7?rWiY=gkPTwI}XZsmSHFuIN|XSX3vGdz&j6_9rdwucOD1agE(v zt3kDAAzT9cb&MnlW*Fr_2daXLG2KYu0EzcbPnyY_JIAe#1)^5#TlvDf^kL`&;veggi|X7YK1oVdzSl4BEjlcX3zihYwVTH{jUWL01p{iQFQmbbRHU zdQ{=aY)F;{b#a}nPvgZ{zt%3xEY95f^2&_;apF6M@Dj$|oQ9sfA-|IsG<2=NHZ}IW zx>zaqnC|hm2SCBLmsYzGxCz)4k|MXpTYIMP9*XUAv1XR6&1wh5DBJd4C=O%yj{Fqd zU%$K{2y8-A^}mv`N{(*8)3fZdpq_ntRnm2JQEk$iQ_`ew)(Sj`|K%ncQBkCdmJD-k zBTx;9%Jqtt8kqL}1=RKlFo_H0xLHWN_N9L$D70CF{73!gaVLM8f`BwV$%9k$&E{6< zjngPME|2SjQJbFq-R${z#7zR+6SP{w4GXnrUmcuB>n!K%;otqo?_dm5XyeMHBn`RK z?SiX}(sY2+=d=z1dtdOiQCt}9GdJ=rO4hN0beLK3DeYpUW)8qxrKF^GoTPS`{%#~# zQIHwN7FX!~=M~%dG1CN`q=2@Y6Ce_Y5x5m3|M{`|034JlcB%`VoK*oacSVy#ee{V$`e$Z0wnU zMTf=iIy&eL3k+=-S?BXffEht)FQ-&M;{1DVP-!g+ zp+?_596?><)N@ixc z%D{Pfa3CRH8s(Is9H^p`g+0pl&Jnm>v^;e*FjLnC#^q2@01g4ZH9LSzP3JK!9Yp|+ z&HJx(5qN==W;2jcdkEy)UEkhe=arYMfok`UIP`1c04&x8l8kpS-iiX~(-9wYwbTd$ zn=>j<2U8+u$u{X;QgZS`fL6NAYpn_~oBab|=Mn~_VA26DTTt4p5@3J3_;?h2%EB$6 zc4=fdjkpmIfagH9dCmLq1twkA`l12))(tA0fMod6Uc=I|VAl7j zp9d7rBNK2m$)j33tJiomtAHXVO*?Ag{Vg&|A`n0DjR?rSx|R-|`4Ym%5Q|Q^H99(K zt@soLl}t4jwMhwy+@gxEN5o}Kxp@;P!4s&2-4%?CGH!&uj|;-7MfDOMLFhXpsbYa8 z(iWyGKpL3@1t9>XV*=o69C%QgAo+_Uf`UH*{BLkgEA^#KzM!mA21qpy@|sgk5Q7aJ zaxE69DYUUM2G|ZLkm?iOzn=#scg$MwT*j&0HkAaOVISXezNtPZdt<}$AuDSXD0biD zGAPP&v_3dN1h;S8<>TmeBo=8#!Wsu=awl`SVt$tjAUnr}pbYeRkS+w6ILq=$ty}uV z9f{@6a~KA7)YX-5t!kVGd#!o}Emj&o@_N%&$lc|Eq5C=p**Sg2zQ5kR4G!)F?&7Z6 z6hR|Rz!5^w|EdnapEo5uvtW#w&;WGXG_&HR>UCV;lEiW_DN*nZFiURm`vlN)KDGyp z$j`9auv&mwhZ%qYhe2sPG7-;mGbnVEZ_(1i%xq&7=#)3-n;R4CXh%B0CA}qk@`&oL zwj7EP6Mm2Usdqd-1p`3w7bRXXiysew2nm~JGEp9GZ>+H0BY4sWaN8JQpKetsr>mRP z1Vr2oU4h7VzA%&5Clfezy!{HWIK%B!`EXLYy2T%ik~u~?c1bhKJOCtiz+VL5*DK1& zFeJLg0fyS*Hz*o7Jx`0I!=Z|8dMI5Duwz$1xHt=ZvqaWg1Nvi3$}NBxORkVT+%!o- z4Oe^}v|Pe}_zu8X4-g2}%cM!g06LTEjj>c&63WUAtUyBvnf8MQO0C;N+1&{mW z0D>5qOh$vN{GC7D8QMbri}U@nmMgfAU@zDwr&zIcr9rYx$x@fzA71Fc36d~OU5+up zy@#O(jUhrZI08Xxpy=az%mL(xzZdrBYgg|j3dU-^lV6NyicyP@7_%(C$2K+=w>bUq z_hZ&!2)@aBTV|5;ZwUT#M$Z_DkfN&!;JD{ii8KEE-_7jrIA6npxGEZ{jNdoO`hM3V zuf4x_L+w6tZc2k9h7X97y8?Cp zu2ra+z*(q&^!6yWRUzVWl`rJq=a~c&sDhHw6aTxvD72vg8p9DeC{?kI6Y+UYh9CZa zqVTi zw{40NOIHzbhjP0c{$AYTEO={q*0(P4zb-cr^$BRsA_ffARt0?ARxEHw?9W5D!$i%= z|3~p>0@qDnilRLijl450hVivg5?o76$@dS~0~&Zh47 Date: Wed, 13 Feb 2019 14:56:38 +0100 Subject: [PATCH 14/48] [docs] Improve documentation for the Deployment Server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Simplified the documentation to improve readability * Split out the API Reference from the original topic * Used the correct qdoc tags where possible * Formatted the tables better for the HTTP API Task-number: AUTOSUITE-752 Change-Id: Iaa308b7f995070993b264ee9bfdcaf2e1bd46d8a Reviewed-by: Topi Reiniö Reviewed-by: Nikolay Zamotaev --- doc/src/deployment-server-reference.qdoc | 447 +++++++++++++++++++++++ doc/src/deployment-server.qdoc | 432 +++------------------- 2 files changed, 488 insertions(+), 391 deletions(-) create mode 100644 doc/src/deployment-server-reference.qdoc diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc new file mode 100644 index 0000000..c49c3d2 --- /dev/null +++ b/doc/src/deployment-server-reference.qdoc @@ -0,0 +1,447 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-reference.html + \previouspage Qt Automotive Suite Deployment Server + \contentspage Qt Automotive Suite + + \title Qt Automotive Suite Deployment Server API Reference + + \section1 API Reference + + The following tables describe the requests, their parameters, as well as the corresponding responses. + + \section2 hello + + Checks whether you are using the correct Platform and the right API to communicate with the deployment + server. + \table + \header + \li Parameter + \li Description + \row + \li platform + \li The platform on which the client is running. This parameter sets the architecture + of the packages you get. For more information, refer to the + \c{settings.APPSTORE_PLATFORM} parameter in the \c{appstore/settings.py} file. + \row + \li version + \li The Deployment Server's HTTP API version that you are using to communicate with the + server. For more information, see \c{settings.APPSTORE_VERSION}. + \row + \li require_tag + \li An optional parameter used to filter packages by tags. Receives a comma-separated + list of tags; these tags must be alphanumeric. Only applications that contain any + of the specified tags should be listed. + \row + \li conflicts_tag + \li An optional parameter used to filter packages by tags. Receives a comma-separated + list of tags; these tags must be alphanumeric. Applications that contain any of the + speicifed tags should be excluded. + \row + \li architecture + \li An optional parameter used to filter packages by architecture. Receives the CPU + architecture. If the architecture is not speciifed, only packages showing \e{All} + architecture are listed. + \endtable + + Returns a JSON object with the following fields and values. + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,5} status + \li ok + \li Successful + \row + \li maintenance + \li The server is in maintenance mode and cannot be used at the moment. + \row + \li incompatible-platform + \li The platform you are using is not compatible. + \row + \li incompatible-version + \li The API version you are using is not compatible. + \row + \li malformed-tag + \li The tag format is incorrect, may not be alphanumeric, or could + not be parsed. + \endtable + + \section2 login + + Logs onto the deployment server with the given username and password. + Either an IMEI or a unique hardware identifier, such as a MAC address, + must be provided. This call is necessary to be able to download apps. + \table + \header + \li Parameter + \li Description + \row + \li username + \li The username. + \row + \li password + \li The password for the specified username. + \endtable + + Returns a JSON object with the following fields and values. + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,4} status + \li ok + \li Login successful. + \row + \li missing-credentials + \li No username or password was provided. + \row + \li account-disabled + \li The account has been disabled in the Admin panel. + \row + \li authentication-failed + \li The user name and/or password may be wrong; or another authentication error + occurred. + \endtable + + \section2 logout + Logs out the currently logged-in user from the deployment server. + + Returns a JSON object with the following fields and values. + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,2} status + \li ok + \li The user has logged out successfully. + \row + \li failed + \li The user was not logged in. + \endtable + + \section2 app/list + Lists all apps. The returned list can be filtered by using the \c{category_id} and the + \c{filter} parameters. + \table + \header + \li Parameter + \li Description + \row + \li category_id + \li Limits the app to those with this category ID only. + \row + \li filter + \li Lists apps with names that match this filter only. + \endtable + + Returns an array of JSON objects (\b{not an object itself!}). + \table + \header + \li JSON Field + \li Description + \row + \li id + \li A unique app ID, in reverse domain name notation. + \row + \li name + \li The app's name. + \row + \li vendor + \li The vendor name for the app; not the vendor ID. + \row + \li category + \li A category name for the app. + \row + \li tags + \li JSON array of app tags + \row + \li version + \li The app's version, returned as a string. If the there is no version number, the + default version, "0.0.0" is returned. + \row + \li architecture + \li The app's architecture, returned as detected in the app's library components. + + If the application is not native, contains \e{All}. + Otherwise it is formed like this: mips-little_endian-32-elf + + Where it is a combination of: + \list 1 + \li CPU architecture, as returned by \l{https://doc.qt.io/qt-5/qsysinfo.html#buildCpuArchitecture}{QsysInfo::buildCpuArchitecture()} + \li CPU endianness, either \c{little_endian} or \c{big_endian}) + \li ABI bitness + \li Binary format, either \c{elf}, \c{mach_o} or \c{pe32} + \endlist + \row + \li briefDescription + \li A short text that describes the app, limited to 1 line, approximately 80-130 + characters. + \row + \li category_id + \li Numeric category ID that matches the app's category. + \endtable + + \section2 app/icon + Returns an icon for the given application id. + \table + \header + \li Parameter + \li Description + \row + \li id + \li The app ID. + \endtable + + Returns a PNG image if the app exists; an HTTP 404 error otherwise. + + \section2 app/description + + Returns a description for the given app ID. + + \table + \header + \li Parameter + \li Description + \row + \li id + \li app ID + \endtable + + Returns a description text for the app, either HTML or plain text. + + \section2 app/purchase + + Returns a URL that you can use to download the requested app for a certain period of time + only; configurable in the settings. + + \note This request is a legacy from the AppStore. Changing the name of this API would + involve changes in Neptune 3 UI. + + \table + \header + \li Parameter + \li Description + \row + \li device_id + \li A unique device ID for the client hardware; currently not used. + \row + \li id + \li The app ID. + \endtable + Returns a JSON object: + \table + \header + \li JSON Field + \li Value + \li Description + \row + \li {1,2} status + \li ok + \li Successful + \row + \li failed + \li + \li An error has occurred, dheck the error field for more information. + \row + \li error + \li Text. + \li If the status is equal to \c{failed}, contains an error description. + \row + \li url + \li A URL. + \li The URL from where to download the app. Expires according to the value specified + in \c{expiresIn}. + \row + \li expiresIn + \li An integer value. + \li Time in seconds during which the download URL is valid. + \endtable + + \section2 category/list + + Lists all of the available categories. Also returns the \e{All} metacategory, that is used + to hold all available applications. + + Returns an array of JSON objects (\b{not an object itself!}). + + \table + \header + \li JSON field + \li Description + \row + \li id + \li Unique category id. + \row + \li name + \li Category name. + \endtable + + \section2 category/icon + Returns an icon for the given category ID. + + \table + \header + \li Parameter + \li Description + \row + \li id + \li The category ID. + \endtable + + Returns an image in PNG format or an empty 1x1 PNG file. + + \note Currently takes the icon of the first app in the category, if it exists. + + \section2 upload + Accepts remote package upload requests. + The user must be in the \e{staff} group to use this API. Also requires either basic authentication + or a previous call to the \c{login} method. This is a POST request to the server due to the parameters used. + \table + \header + \li Parameter + \li Description + \row + \li description + \li Package description, long version. Can be text or HTML. + \row + \li short-description + \li One line package description. + \row + \li category + \li Category name for the category where the package will be put. + \row + \li vendor + \li Vendor name for the package. + \row + \li package + \li Package itself. This is uploaded as a file parameter. + \endtable + + Returns JSON object: + \table + \header + \li Parameter + \li Value + \li Description + \row + \li {1,9} status + \li ok + \li Success + \row + \li no description + \li The description parameter is missing. + \row + \li no short description + \li The short-description parameter is missing. + \row + \li no category + \li The category parameter is missing. + \row + \li no vendor + \li The vendor parameter is missing. + \row + \li Package validation failed + \li Package did not pass format or sanity validation + \row + \li Non-existing category + \li The specified category does not match the parameter passed. + \row + \li Non-existing vendor + \li The specified vendor does not match the parameter passed. + \row + \li no package to upload + \li There was no \c{package} parameter in the request, or it was not a POST request. + \endtable + + \section2 API Usage Examples + + The Deployment Server exposes an HTTP API. Arguments to these requests need to be provided + using the HTTP GET or POST syntax. The data is returned in JSON, PNG, or text format, depending + on the request. + + \section3 Workflow + + \list 1 + \li Send a \c{hello} request to the server to get the current status and to check + whether your platform is compatible with this server instance: + + \tt http:///hello?platform=AM&version=1 + + Returns: + + \tt { { "status": "ok" } } + + \li Login as \c{user} with password, \c{pass}: + + \tt http:///login?username=user&password=pass + + Returns: + + \tt { { "status": "ok" } } + + \li List all applications + + \tt http:///app/list + + Returns: + + \tt { [{ "category": "Entertainment", + "name": "Nice App", + "vendor": "Luxoft", + "briefDescription": "Nice App is a really nice app.", + "category_id": 4, + "id": "com.luxoft.niceapp"}, + .... + ] } + + + \li Request a download for an app: + + \tt http:///app/purchase?device_id=12345&id=com.luxoft.niceapp + + Returns: + + \tt { { "status": "ok", + "url": "http:///app/download/com.luxoft.niceapp.2.npkg", + "expiresIn": 600 + } } + + \li Use the \c{url} provided in step 4 to download the application within \c{expiresIn} + seconds. + + \endlist +*/ \ No newline at end of file diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index d3f2e81..c6b2c45 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -28,6 +28,9 @@ /*! \page qtauto-deployment-server-index.html + \contentspage {Qt Automotive Suite} + \nextpage Qt Automotive Suite Deployment Server API Reference + \title Qt Automotive Suite Deployment Server The Qt Automotive Suite Deployment Server is a new component in the Qt Automotive Suite 5.12. @@ -91,421 +94,68 @@ version "0.0.0" is assigned. \endlist - \section1 API - \section2 hello - Checks whether you are using the correct Platform and the right API to communicate with the deployment server. - \table - \header - \li Parameter - \li Description - \row - \li platform - \li The platform the client is running on. This sets the architecture of the packages - you get (see \b{settings.APPSTORE_PLATFORM} parameter in \b{appstore/settings.py} file). - \row - \li version - \li The Deployment Server HTTP API version you are using to communicate with the server - (see \b{settings.APPSTORE_VERSION}). - \row - \li require_tag - \li Optional parameter for filtering packages by tags. Receives a comma-separated list of tags. - Only applications containing any of the specified tags should be listed. - Tags must be alphanumeric. - \row - \li conflicts_tag - \li Optional parameter for filtering packages by tags. Receives a comma-separated list of tags. - No application containing any of the tags should be listed. Tags must be alphanumeric. - \row - \li architecture - \li Optional parameter for filtering packages by architecture. Receives the CPU architecture. - If architecture is not specified, only packages showing \e{All} architecture are listed. - \endtable - Returns a JSON object: - \table - \header - \li JSON field - \li Value - \li Description - \row - \li {1,5} status - \li ok - \li Successful - \row - \li maintenance - \li Server is in maintenance mode and can not be used at the moment. - \row - \li incompatible-platform - \li You are using an incompatible platform. - \row - \li incompatible-version - \li You are using incompatible version of the API. - \row - \li malformed-tag - \li Tag had wrong format, was not alphanumeric or could not be parsed. - \endtable - \section2 login - Logs onto the deployment server with the given username and password. - Either an IMEI or a \l {https://en.wikipedia.org/wiki/MAC_address} {unique hardware identifier} must be provided. This call is needed for downloading apps. - \table - \header - \li Parameter - \li Description - \row - \li username - \li The username - \row - \li password - \li Password for the given username - \endtable - Returns a JSON object: - \table - \header - \li JSON field - \li Value - \li Description - \row - \li {1,4} status - \li ok - \li Login successful. - \row - \li missing-credentials - \li No username or password provided - \row - \li account-disabled - \li Account is disabled in the admin panel - \row - \li authentication-failed - \li Wrong username and/or password or other authentication error - \endtable - \section2 logout - Logs out the currently logged-in user from the deployment server. - - Returns a JSON object: - \table - \header - \li JSON field - \li Value - \li Description - \row - \li {1,2} status - \li ok - \li Successfully logged out - \row - \li failed - \li Was not logged in - \endtable - - \section2 app/list - Lists all apps. The returned list can be filtered by using the \b{category_id} and the \b{filter} arguments. - \table - \header - \li Parameter - \li Description - \row - \li category_id - \li Limits applications to the category with this id. - \row - \li filter - \li Only lists apps whose names match the filter. - \endtable - Returns an array of JSON objects (\b{not an object itself!}). - \table - \header - \li JSON field - \li Description - \row - \li id - \li Unique application id. In reverse domain name notation - \row - \li name - \li Application name - \row - \li vendor - \li Vendor name for application (not vendor id) - \row - \li category - \li Category name for application - \row - \li tags - \li JSON array of application tags - \row - \li version - \li Application version. Returned as a string. If the application information lacks - a version number, “0.0.0” is returned - \row - \li architecture - \li Application architecture. Returned as detected in the library components of the application - - If the application is not native, contains \e{All}. - Otherwise it is formed like this: mips-little_endian-32-elf - - Where it is a combination of: - \list 1 - \li CPU architecture, as returned by QsysInfo::buildCpuArchitecture() - \li CPU endianness (either \b{little_endian} or \b{big_endian}) - \li ABI bitness - \li binary format (\b{elf}, \b{mach_o} or \b{pe32}) - \endlist - - \row - \li briefDescription - \li Short textual app description (should be limited to 1 line, around 80-130 characters) - \row - \li category_id - \li Numeric category id matching the application category field. - \endtable - \section2 app/icon - Returns an icon for the given application id. - \table - \header - \li Parameter - \li Description - \row - \li id - \li Application id - \endtable - Returns a PNG image or a 404 error, if the application does not exist. - \section2 app/description - Returns a description for the given application id. - \table - \header - \li Parameter - \li Description - \row - \li id - \li Application id - \endtable - Returns an application description text - either HTML or plain text. - \section2 app/purchase - Returns a URL which can be used for downloading the requested application for certain period of time (configurable in the settings). - \note This is legacy from AppStore. Changing the name of this API would involve changes in reference UI. - The real action is preparing the package for download. For discussion, contact nzamotaev@luxoft.com. - \table - \header - \li Parameter - \li Description - \row - \li device_id - \li Unique device id for client hardware (currently unused) - \row - \li id - \li Application id - \endtable - Returns a JSON object: - \table - \header - \li JSON field - \li Value - \li Description - \row - \li {1,2} status - \li ok - \li Successful - \row - \li failed - \li Something went wrong. Check the error field for more information. - \row - \li error - \li text - \li If status equals to \e{failed}, contains an error description. - \row - \li url - \li URL - \li URL for downloading the application. - Expires in the time specified in the \c{expiresIn} field. - \row - \li expiresIn - \li Integer value - \li Time in seconds for which the download URL stays valid. - \endtable - - \section2 category/list - Lists all the available categories. Also returns \e{All} metacategory, that is used to hold all available applications. - Returns an array of JSON objects (\b{not an object itself!}). - \table - \header - \li JSON field - \li Description - \row - \li id - \li Unique category id. - \row - \li name - \li Category name. - \endtable - \section2 category/icon - Returns an icon for the given category id. - \table - \header - \li Parameter - \li Description - \row - \li id - \li Id of the category. - \endtable - Returns a PNG image or an empty 1x1 PNG file. - \note Currently takes the icon of the first application in the category if it exists. - This should be fixed. - \section2 upload - Accepts remote package upload requests. - The user must be in the \e{staff} group to use this API. Also requires either basic authentication - or a previous call to the \c{login} method. This is a POST request to the server due to the parameters used. - \table - \header - \li Parameter - \li Description - \row - \li description - \li Package description, long version. Can be text or HTML. - \row - \li short-description - \li One line package description. - \row - \li category - \li Category name for the category where the package will be put. - \row - \li vendor - \li Vendor name for the package. - \row - \li package - \li Package itself. This is uploaded as a file parameter. - \endtable - Returns JSON object: - \table - \header - \li Parameter - \li Value - \li Description - \row - \li {1,9} status - \li ok - \li Success - \row - \li no description - \li Description parameter missing - \row - \li no short description - \li Short-description parameter missing - \row - \li no category - \li Category parameter missing - \row - \li no vendor - \li Vendor parameter missing - \row - \li Package validation failed - \li Package did not pass format or sanity validation - \row - \li Non-existing category - \li No category matches the passed parameter - \row - \li Non-existing vendor - \li No vendor matches passed parameter - \row - \li no package to upload - \li There was no \c{package} parameter in the request, or it was not a POST request - \endtable - - \section2 API use examples - The deployment server exposes a HTTP API to the world. Arguments to the functions need to be provided using the - HTTP GET/POST syntax. Returned data will be in JSON, PNG or text format, depending on the function. - \section3 Workflow - \list 1 - \li Send a \b{hello} request to the server to get the current status and to check - whether your platform is compatible with this server instance: - - \tt http:///hello?platform=AM&version=1 - - Returns: - - \tt { { "status": "ok" } } - - \li Login as user \e{user} with password \e{pass}: - - \tt http:///login?username=user&password=pass - - Returns: - - \tt { { "status": "ok" } } - - \li List all applications - - \tt http:///app/list - - Returns: - - \tt { [{ "category": "Entertainment", - "name": "Nice App", - "vendor": "Luxoft", - "briefDescription": "Nice App is a really nice app.", - "category_id": 4, - "id": "com.luxoft.niceapp"}, - .... - ] } - - - \li Request a download for an app: - - \tt http:///app/purchase?device_id=12345&id=com.luxoft.niceapp - - Returns: - - \tt { { "status": "ok", - "url": "http:///app/download/com.luxoft.niceapp.2.npkg", - "expiresIn": 600 - } } + \section1 Installation - \li Use the \b{url} provided in step 4 to download the application within - \b{expiresIn} seconds. + \section2 Set up the Server in a Virtual Environment - \endlist + Before you install the dependencies in the Python virtual environment, you must install the + \c{libffi-dev} package. Next, prepare the virtual environment: - \section1 Installation - \section2 Setting up the Server in virtualenv - Before installing dependencies in the Python virtual environment, libffi-dev package must be - installed. After package installation, prepare the virtual environment: \code virtualenv ./venv ./venv/bin/pip install -r requirements.txt \endcode - Before running the server, make sure to adapt the \b{APPSTORE_*} settings in \b{appstore/settings.py} to your environment. + Make sure to adapt the \c{APPSTORE_*} settings in \c{appstore/settings.py} to your environment, + before you run the server. + + Since package downloads are done via temporary files, you need to setup a cron-job to remove + these temporary files periodically. The cron-job should be triggered every + \c{settings.APPSTORE_DOWNLOAD_EXPIRY/2} minutes; it needs to run: - Since package downloads are done via temporary files, one needs to setup a cron-job to cleanup - these temporary files every now and then. The job should be triggered every \b{settings.APPSTORE_DOWNLOAD_EXPIRY/2} - minutes and it just needs to execute: \code ./manage.py expire-downloads \endcode - \section2 Starting the server - Running the server: + + \section2 Start the Server + + To start the server, run the following command in your terminal: + \code ./manage.py runserver 0.0.0.0:8080 \endcode - will start the server on port 8080, reachable by anyone. One can tweak the listening address to whatever fits the needs. - \section2 Maintenance tasks + + This command starts the server on port 8080, and is reachable by anyone. You can modify the + listening address to another address that suits your use case. + + \section2 Maintain the Server + \list - \li Cleaning up the downloads directory: + \li Clean up the downloads directory: \code ./manage.py expire-downloads \endcode - will remove all files from the downloads directory, that are older than - \b{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. This should be called from a cron-job (see above). - \li Manually verifying a package for upload: + + This command removes all files from the downloads directory, that are older than + \c{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. Ideally, this command should be run via a cron-job. + + \li Manually verify a package for upload: + \code ./manage.py verify-upload-package \endcode - will tell if \b{} is a valid package that can be uploaded to the store. - \li Manually adding a store signature to a package: + + This command verifies if \c{} is a valid package that can be uploaded to the Downloads + app. + + \li Manually add a store signature to a package: + \code ./manage.py store-sign-package [device id] \endcode - will first verify \b{}. If this succeeds, it will copy \b{} to \b{} and - add a store signature. The optional \b{[device id]} parameter will lock the generated package to the device with this id. - \endlist - \section1 Architecture - This is a django application. - - \b{TBD} + This command first verifies the \c{}. If this verification succeeds, \c{} + is copied over to \c{} and a store signature is added. The optional \c{[device id]} + parameter locks the generated package to the device with this ID. + \endlist */ From 962bbcbcdab445169952c0a98c911d08ec564529 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Mon, 25 Mar 2019 17:54:52 +0300 Subject: [PATCH 15/48] Django version update. As a prerequisite to category icon implementation Task-number: AUTOSUITE-759 Change-Id: I2bf1f593c9f5cdeb6d77527945d35446ac23c8a8 Reviewed-by: Dominik Holland --- appstore/settings.py | 22 ++++++++++++-- appstore/urls.py | 33 ++++++++++---------- requirements.txt | 2 +- store/admin.py | 10 +++--- store/api.py | 52 +++++++++++++++++--------------- store/migrations/0001_initial.py | 31 ++++++++----------- store/utilities.py | 6 ++++ 7 files changed, 88 insertions(+), 68 deletions(-) diff --git a/appstore/settings.py b/appstore/settings.py index a405491..e3edabc 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -64,10 +64,28 @@ # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -TEMPLATE_DEBUG = True - ALLOWED_HOSTS = [] +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': ( + "django.contrib.auth.context_processors.auth", + "django.template.context_processors.debug", + "django.template.context_processors.i18n", + "django.template.context_processors.media", + "django.template.context_processors.static", + "django.template.context_processors.tz", + "django.contrib.messages.context_processors.messages" + ), + 'debug': True + }, + }, +] + # Application definition diff --git a/appstore/urls.py b/appstore/urls.py index 4d0467f..7c99a6a 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -30,32 +30,33 @@ ## ############################################################################# -from django.conf.urls import patterns, include, url +from django.conf.urls import include, url from django.contrib import admin +from store import api as store_api import settings -base_urlpatterns = patterns('', +base_urlpatterns = [ url(r'^admin/', include(admin.site.urls)), - url(r'^hello$', 'store.api.hello'), - url(r'^login$', 'store.api.login'), - url(r'^logout$', 'store.api.logout'), - url(r'^app/list$', 'store.api.appList'), - url(r'^app/icon', 'store.api.appIcon'), - url(r'^app/description', 'store.api.appDescription'), - url(r'^app/purchase', 'store.api.appPurchase'), - url(r'^app/download/(.*)$', 'store.api.appDownload'), - url(r'^category/list$', 'store.api.categoryList'), - url(r'^category/icon$', 'store.api.categoryIcon'), - url(r'^upload$', 'store.api.upload'), -) + url(r'^hello$', store_api.hello), + url(r'^login$', store_api.login), + url(r'^logout$', store_api.logout), + url(r'^app/list$', store_api.appList), + url(r'^app/icon', store_api.appIcon), + url(r'^app/description', store_api.appDescription), + url(r'^app/purchase', store_api.appPurchase), + url(r'^app/download/(.*)$', store_api.appDownload), + url(r'^category/list$', store_api.categoryList), + url(r'^category/icon$', store_api.categoryIcon), + url(r'^upload$', store_api.upload), +] prefix = '^' if settings.URL_PREFIX !='': prefix = prefix + settings.URL_PREFIX + '/' -urlpatterns = patterns('', +urlpatterns = [ url(prefix, include(base_urlpatterns)), -) +] diff --git a/requirements.txt b/requirements.txt index d1eb087..79be93a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pkg-resources==0.0.0 PyYAML -django==1.7.9 +django==1.11 django-common pyOpenSSL M2Crypto diff --git a/store/admin.py b/store/admin.py index eff2550..f2adb23 100644 --- a/store/admin.py +++ b/store/admin.py @@ -34,7 +34,7 @@ from django import forms from django.conf import settings -from django.conf.urls import patterns +from django.conf.urls import include, url from django.contrib import admin from django.core.exceptions import PermissionDenied from django.shortcuts import redirect, get_object_or_404 @@ -87,10 +87,10 @@ def move(self, obj): def get_urls(self): admin_view = self.admin_site.admin_view - urls = patterns('', - (r'^(?P\d+)/move_up/$', admin_view(self.move_up)), - (r'^(?P\d+)/move_down/$', admin_view(self.move_down)), - ) + urls = [ + url(r'^(?P\d+)/move_up/$', admin_view(self.move_up)), + url(r'^(?P\d+)/move_down/$', admin_view(self.move_down)), + ] return urls + super(CategoryAdmin, self).get_urls() def move_up(self, request, item_pk): diff --git a/store/api.py b/store/api.py index 88ceef4..63c3121 100644 --- a/store/api.py +++ b/store/api.py @@ -37,13 +37,13 @@ from django.db.models import Q, Count from django.http import HttpResponse, HttpResponseForbidden, Http404, JsonResponse from django.contrib import auth -from django.template import Context, loader from django.views.decorators.csrf import csrf_exempt from authdecorators import logged_in_or_basicauth, is_staff_member from models import App, Category, Vendor, savePackageFile from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage from utilities import packagePath, iconPath, downloadPath +from utilities import getRequestDictionary from osandarch import normalizeArch from tags import SoftwareTagList @@ -53,20 +53,20 @@ def hello(request): if settings.APPSTORE_MAINTENANCE: status = 'maintenance' - elif request.REQUEST.get("platform", "") != str(settings.APPSTORE_PLATFORM_ID): + elif getRequestDictionary(request).get("platform", "") != str(settings.APPSTORE_PLATFORM_ID): status = 'incompatible-platform' - elif request.REQUEST.get("version", "") != str(settings.APPSTORE_PLATFORM_VERSION): + elif getRequestDictionary(request).get("version", "") != str(settings.APPSTORE_PLATFORM_VERSION): status = 'incompatible-version' for j in ("require_tag", "conflicts_tag",): - if j in request.REQUEST: #Tags are coma-separated, + if j in getRequestDictionary(request): #Tags are coma-separated, versionmap = SoftwareTagList() - if not versionmap.parse(request.REQUEST[j]): + if not versionmap.parse(getRequestDictionary(request)[j]): status = 'malformed-tag' break request.session[j] = str(versionmap) - if 'architecture' in request.REQUEST: - request.session['architecture'] = normalizeArch(request.REQUEST['architecture']) + if 'architecture' in getRequestDictionary(request): + request.session['architecture'] = normalizeArch(getRequestDictionary(request)['architecture']) else: request.session['architecture'] = '' return JsonResponse({'status': status}) @@ -74,11 +74,10 @@ def hello(request): def login(request): status = 'ok' - try: try: - username = request.REQUEST["username"] - password = request.REQUEST["password"] + username = getRequestDictionary(request)["username"] + password = getRequestDictionary(request)["password"] except KeyError: raise Exception('missing-credentials') @@ -114,19 +113,19 @@ def upload(request): status = 'ok' try: try: - description = request.REQUEST["description"] + description = getRequestDictionary(request)["description"] except: raise Exception('no description') try: - shortdescription = request.REQUEST["short-description"] + shortdescription = getRequestDictionary(request)["short-description"] except: raise Exception('no short description') try: - category_name = request.REQUEST["category"] + category_name = getRequestDictionary(request)["category"] except: raise Exception('no category') try: - vendor_name = request.REQUEST["vendor"] + vendor_name = getRequestDictionary(request)["vendor"] except: raise Exception('no vendor') @@ -159,10 +158,10 @@ def upload(request): def appList(request): apps = App.objects.all() - if 'filter' in request.REQUEST: - apps = apps.filter(name__contains = request.REQUEST['filter']) - if 'category_id' in request.REQUEST: - catId = request.REQUEST['category_id'] + if 'filter' in getRequestDictionary(request): + apps = apps.filter(name__contains = getRequestDictionary(request)['filter']) + if 'category_id' in getRequestDictionary(request): + catId = getRequestDictionary(request)['category_id'] if catId != -1: # All metacategory apps = apps.filter(category__exact = catId) @@ -223,27 +222,29 @@ def appDescription(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) + appId = getRequestDictionary(request)['id'] try: - app = App.objects.get(appid__exact = request.REQUEST['id'], architecture__in = archlist).order_by('architecture') + app = App.objects.get(appid__exact = appId, architecture__in = archlist).order_by('architecture') app = app.last() return HttpResponse(app.description) except: - raise Http404('no such application: %s' % request.REQUEST['id']) + raise Http404('no such application: %s' % appId) def appIcon(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) + appId = getRequestDictionary(request)['id'] try: - app = App.objects.filter(appid__exact = request.REQUEST['id'], architecture__in = archlist).order_by('architecture') + app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture') app = app.last() with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: response = HttpResponse(content_type = 'image/png') response.write(iconPng.read()) return response except: - raise Http404('no such application: %s' % request.REQUEST['id']) + raise Http404('no such application: %s' % appId) def appPurchase(request): @@ -253,14 +254,14 @@ def appPurchase(request): if 'architecture' in request.session: archlist.append(request.session['architecture']) try: - deviceId = str(request.REQUEST.get("device_id", "")) + deviceId = str(getRequestDictionary(request).get("device_id", "")) if settings.APPSTORE_BIND_TO_DEVICE_ID: if not deviceId: return JsonResponse({'status': 'failed', 'error': 'device_id required'}) else: deviceId = '' - app = App.objects.filter(appid__exact = request.REQUEST['id'], architecture__in=archlist).order_by('architecture') + app = App.objects.filter(appid__exact = getRequestDictionary(request)['id'], architecture__in=archlist).order_by('architecture') app = app.last() fromFilePath = packagePath(app.appid, app.architecture) @@ -314,10 +315,11 @@ def categoryList(request): def categoryIcon(request): response = HttpResponse(content_type = 'image/png') + categoryId = getRequestDictionary(request)['id'] # there are no category icons (yet), so we just return the icon of the first app in this category try: - app = App.objects.filter(category__exact = request.REQUEST['id']).order_by('-dateModified')[0] #FIXME - the category icon is unimplemented + app = App.objects.filter(category__exact = categoryId).order_by('-dateModified')[0] #FIXME - the category icon is unimplemented with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: response.write(iconPng.read()) except: diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index 51b0677..bde513e 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -31,24 +31,28 @@ ## ############################################################################# +# Generated by Django 1.11 on 2019-03-25 14:51 from __future__ import unicode_literals -from django.db import models, migrations from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion import store.models class Migration(migrations.Migration): + initial = True + dependencies = [ - ('auth', '0001_initial'), + ('auth', '0008_alter_user_username_max_length'), ] operations = [ migrations.CreateModel( name='App', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('appid', models.CharField(max_length=200)), ('name', models.CharField(max_length=200)), ('file', models.FileField(storage=store.models.OverwriteStorage(), upload_to=store.models.content_file_name)), @@ -60,43 +64,32 @@ class Migration(migrations.Migration): ('architecture', models.CharField(default=b'All', max_length=20)), ('version', models.CharField(default=b'0.0.0', max_length=20)), ], - options={ - }, - bases=(models.Model,), ), migrations.CreateModel( name='Category', fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=200)), - ('rank', models.SmallIntegerField(unique=True, db_index=True)), + ('rank', models.SmallIntegerField(db_index=True, unique=True)), ], - options={ - }, - bases=(models.Model,), ), migrations.CreateModel( name='Vendor', fields=[ - ('user', models.ForeignKey(primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)), ('name', models.CharField(max_length=200)), ('certificate', models.TextField(max_length=8000)), ], - options={ - }, - bases=(models.Model,), ), migrations.AddField( model_name='app', name='category', - field=models.ForeignKey(to='store.Category'), - preserve_default=True, + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='store.Category'), ), migrations.AddField( model_name='app', name='vendor', - field=models.ForeignKey(to='store.Vendor'), - preserve_default=True, + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='store.Vendor'), ), migrations.AlterUniqueTogether( name='app', diff --git a/store/utilities.py b/store/utilities.py index 42f78f7..8a72661 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -57,6 +57,12 @@ def makeTagList(pkgdata): taglist.append(SoftwareTag(i)) return str(taglist) +def getRequestDictionary(request): + if request.method == "POST": + return request.POST + else: + return request.GET + def packagePath(appId = None, architecture = None): path = settings.MEDIA_ROOT + 'packages/' if (appId is not None) and (architecture is not None): From 75a0b2cfd442c5499bb520eee6131263763e8687 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 12 Feb 2019 18:14:39 +0300 Subject: [PATCH 16/48] Category icon upload implementation Task-number: AUTOSUITE-759 Change-Id: I1030d4b127b41cccfee545e1d4412e61e67f8fc1 Reviewed-by: Egor Nemtsev --- appstore/settings.py | 8 +++ requirements.txt | 2 + store/admin.py | 94 ++++++++++++------------------ store/api.py | 27 ++++++--- store/migrations/0001_initial.py | 7 ++- store/models.py | 85 ++++++++++----------------- store/static/img/category_All.png | Bin 0 -> 620 bytes 7 files changed, 101 insertions(+), 122 deletions(-) create mode 100644 store/static/img/category_All.png diff --git a/appstore/settings.py b/appstore/settings.py index e3edabc..8e67698 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -96,6 +96,7 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'ordered_model', 'store', ) @@ -153,3 +154,10 @@ # trailing slash if there is a path component (optional in other cases). # Examples: "http://media.lawrence.com", "http://example.com/media/" MEDIA_URL = '' + +# Icon size (icons are resized to this size on upload) +ICON_SIZE_X = 36 +ICON_SIZE_Y = 32 +# If the icon should be transformed to monochrome, with alpha channel, when uploaded or not +ICON_DECOLOR = True + diff --git a/requirements.txt b/requirements.txt index 79be93a..41de88c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ pkg-resources==0.0.0 PyYAML django==1.11 django-common +django-ordered-model==2.1 pyOpenSSL M2Crypto Enum34 @@ -9,4 +10,5 @@ ipaddress cffi paramiko cryptography +pillow python-magic==0.4.15 diff --git a/store/admin.py b/store/admin.py index f2adb23..61d5ff0 100644 --- a/store/admin.py +++ b/store/admin.py @@ -33,34 +33,51 @@ import os from django import forms -from django.conf import settings -from django.conf.urls import include, url from django.contrib import admin -from django.core.exceptions import PermissionDenied -from django.shortcuts import redirect, get_object_or_404 from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy +from ordered_model.admin import OrderedModelAdmin +from django.core.files.uploadedfile import InMemoryUploadedFile from store.models import * from utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList +import StringIO class CategoryAdminForm(forms.ModelForm): class Meta: exclude = ["id", "rank"] def save(self, commit=False): - m = super(CategoryAdminForm, self).save(commit) - try: - test = Category.objects.all().order_by('-rank')[:1].values('rank')[0]['rank'] + 1 - except: - test = 0 - m.rank = test + m = super(CategoryAdminForm, self).save(commit=False) return m -class CategoryAdmin(admin.ModelAdmin): + def clean(self): + cleaned_data = super(CategoryAdminForm, self).clean() + #Icon fixing (resize, turn monochrome, add alpha channel) + if cleaned_data['icon']: + if settings.ICON_DECOLOR: + # make image monochrome + alpha channel, this is done to compensate for + # how icons are treated in neptune3-ui + im = Image.open(cleaned_data['icon']) + grey, alpha = im.convert('LA').split() + grey = ImageChops.invert(grey) + im.putalpha(grey) + im = im.convert('LA') + else: + # No conversion, icons are uploaded as-is, only scaling is used. + im = Image.open(cleared_data['icon']) + size = (settings.ICON_SIZE_X,settings.ICON_SIZE_Y,) + im.thumbnail(size, Image.ANTIALIAS) + imagefile = StringIO.StringIO() + im.save(imagefile, format='png') + imagefile.seek(0) + cleaned_data['icon'] = InMemoryUploadedFile(imagefile, 'icon', "icon.png", 'image/png', imagefile.len, None) + return cleaned_data + +class CategoryAdmin(OrderedModelAdmin): form = CategoryAdminForm - list_display = ('name', 'move') - ordering = ('rank',) + list_display = ('name', 'icon_image', 'move_up_down_links') + ordering = ('order',) def save_model(self, request, obj, form, change): obj.save() @@ -70,53 +87,14 @@ def name(self, obj): return obj.name name.short_description = ugettext_lazy('Item caption') - def move(self, obj): - """ - Returns html with links to move_up and move_down views. - """ - button = u' %s' - prefix = settings.STATIC_URL - - link = '%d/move_up/' % obj.pk - html = button % (link, prefix, 'up', _('up')) + " | " - link = '%d/move_down/' % obj.pk - html += button % (link, prefix, 'down', _('down')) + def icon_image(self, obj): + prefix = settings.URL_PREFIX + image_request = prefix + "/category/icon?id=%s" % (obj.id) + html = u'' % (settings.ICON_SIZE_X, settings.ICON_SIZE_Y, image_request) return html - move.allow_tags = True - move.short_description = ugettext_lazy('Move') - - def get_urls(self): - admin_view = self.admin_site.admin_view - urls = [ - url(r'^(?P\d+)/move_up/$', admin_view(self.move_up)), - url(r'^(?P\d+)/move_down/$', admin_view(self.move_down)), - ] - return urls + super(CategoryAdmin, self).get_urls() - - def move_up(self, request, item_pk): - """ - Decrease rank (change ordering) of the menu item with - id=``item_pk``. - """ - if self.has_change_permission(request): - item = get_object_or_404(Category, pk=item_pk) - item.decrease_rank() - else: - raise PermissionDenied - return redirect('admin:store_category_changelist') - - def move_down(self, request, item_pk): - """ - Increase rank (change ordering) of the menu item with - id=``item_pk``. - """ - if self.has_change_permission(request): - item = get_object_or_404(Category, pk=item_pk) - item.increase_rank() - else: - raise PermissionDenied - return redirect('admin:store_category_changelist') + icon_image.allow_tags = True + icon_image.short_description = ugettext_lazy('Category icon') class AppAdminForm(forms.ModelForm): diff --git a/store/api.py b/store/api.py index 63c3121..1c26944 100644 --- a/store/api.py +++ b/store/api.py @@ -50,7 +50,6 @@ def hello(request): status = 'ok' - if settings.APPSTORE_MAINTENANCE: status = 'maintenance' elif getRequestDictionary(request).get("platform", "") != str(settings.APPSTORE_PLATFORM_ID): @@ -308,7 +307,7 @@ def appDownload(request, path): def categoryList(request): # this is not valid JSON, since we are returning a list! allmeta = [{'id': -1, 'name': 'All'}, ] #All metacategory - categoryobject = Category.objects.all().order_by('rank').values('id', 'name') + categoryobject = Category.objects.all().order_by('order').values('id', 'name') categoryobject=allmeta + list(categoryobject) return JsonResponse(categoryobject, safe = False) @@ -316,14 +315,26 @@ def categoryList(request): def categoryIcon(request): response = HttpResponse(content_type = 'image/png') categoryId = getRequestDictionary(request)['id'] - - # there are no category icons (yet), so we just return the icon of the first app in this category try: - app = App.objects.filter(category__exact = categoryId).order_by('-dateModified')[0] #FIXME - the category icon is unimplemented - with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: - response.write(iconPng.read()) + if categoryId != '-1': + category = Category.objects.filter(id__exact = categoryId)[0] + filename = iconPath() + "category_" + str(category.id) + ".png" + else: + from django.contrib.staticfiles import finders + filename = finders.find('img/category_All.png') + with open(filename, 'rb') as icon: + response.write(icon.read()) + response['Content-Length'] = icon.tell() + except: - emptyPng = '\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x01\x03\x00\x00\x00%\xdbV\xca\x00\x00\x00\x03PLTE\x00\x00\x00\xa7z=\xda\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\nIDAT\x08\xd7c`\x00\x00\x00\x02\x00\x01\xe2!\xbc3\x00\x00\x00\x00IEND\xaeB`\x82' + # In case there was error in searching for category, + # return this image: + # +-----+ + # | | + # |Error| + # | | + # +-----+ + emptyPng = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x01\x03\x00\x00\x00I\xb4\xe8\xb7\x00\x00\x00\x06PLTE\x00\x00\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\x33IDAT\x08\xd7\x63\xf8\x0f\x04\x0c\x0d\x0c\x0c\x8c\x44\x13\x7f\x40\xc4\x01\x10\x71\xb0\xf4\x5c\x2c\xc3\xcf\x36\xc1\x44\x86\x83\x2c\x82\x8e\x48\xc4\x5f\x16\x3e\x47\xd2\x0c\xc5\x46\x80\x9c\x06\x00\xa4\xe5\x1d\xb4\x8e\xae\xe8\x43\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82" response.write(emptyPng) return response diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index bde513e..dca1599 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -69,9 +69,14 @@ class Migration(migrations.Migration): name='Category', fields=[ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('order', models.PositiveIntegerField(db_index=True, editable=False)), ('name', models.CharField(max_length=200)), - ('rank', models.SmallIntegerField(db_index=True, unique=True)), + ('icon', models.ImageField(storage=store.models.OverwriteStorage(), upload_to=store.models.category_file_name)), ], + options={ + 'ordering': ('order',), + 'abstract': False, + }, ), migrations.CreateModel( name='Vendor', diff --git a/store/models.py b/store/models.py index eb3ede9..08bebc3 100644 --- a/store/models.py +++ b/store/models.py @@ -31,69 +31,50 @@ ############################################################################# import os +from PIL import Image, ImageChops from django.db import models +from ordered_model.models import OrderedModel from django.conf import settings from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage +from django.db.models.fields.files import ImageFieldFile from utilities import packagePath, writeTempIcon, makeTagList +def category_file_name(instance, filename): + # filename parameter is unused. See django documentation for details: + # https://docs.djangoproject.com/en/1.11/ref/models/fields/#django.db.models.FileField.upload_to + return settings.MEDIA_ROOT + "icons/category_" + str(instance.id) + ".png" -class Category(models.Model): +class OverwriteStorage(FileSystemStorage): + def get_available_name(self, name, max_length=None): + if self.exists(name): + os.remove(os.path.join(settings.MEDIA_ROOT, name)) + return name + +class Category(OrderedModel): name = models.CharField(max_length = 200) - rank = models.SmallIntegerField(unique = True, db_index = True) + icon = models.ImageField(upload_to = category_file_name, storage = OverwriteStorage()) + + class Meta(OrderedModel.Meta): + ordering = ('order',) def __unicode__(self): return self.name - def is_first(self): - """ - Returns ``True`` if item is the first one in the menu. - """ - return Category.objects.filter(rank__lt = self.rank).count() == 0 - - def is_last(self): - """ - Returns ``True`` if item is the last one in the menu. - """ - return Category.objects.filter(rank__gt = self.rank).count() == 0 - - def increase_rank(self): - """ - Changes position of this item with the next item in the - menu. Does nothing if this item is the last one. - """ - try: - next_item = Category.objects.filter(rank__gt = self.rank)[0] - except IndexError: - pass - else: - self.swap_ranks(next_item) - - def decrease_rank(self): - """ - Changes position of this item with the previous item in the - menu. Does nothing if this item is the first one. - """ - try: - list = Category.objects.filter(rank__lt = self.rank).reverse() - prev_item = list[len(list) - 1] - except IndexError: - pass - else: - self.swap_ranks(prev_item) - - def swap_ranks(self, other): - """ - Swap positions with ``other`` menu item. - """ - maxrank = 5000 - prev_rank, self.rank = self.rank, maxrank - self.save() - self.rank, other.rank = other.rank, prev_rank - other.save() - self.save() + def save(self, *args, **kwargs): + if self.id is None: + # This is a django hack. When category icon is saved and then later accessed, + # category_id is used as a unique icon identifier. When category is first created, + # but not saved yet, category_id is None. So this hack first saves category without icon + # and then saves the icon separately. This is done to prevent creation of category_None.png + # file, when the icon is saved. + saved_icon = self.icon + self.icon = None + super(Category, self).save(*args, **kwargs) + self.icon = saved_icon + super(Category, self).save(*args, **kwargs) class Vendor(models.Model): user = models.ForeignKey(User, primary_key = True) @@ -104,12 +85,6 @@ def __unicode__(self): return self.name -class OverwriteStorage(FileSystemStorage): - def get_available_name(self, name): - if self.exists(name): - os.remove(os.path.join(settings.MEDIA_ROOT, name)) - return name - def content_file_name(instance, filename): return packagePath(instance.appid, instance.architecture) diff --git a/store/static/img/category_All.png b/store/static/img/category_All.png new file mode 100644 index 0000000000000000000000000000000000000000..2f44f41939728dcbb5727c56454d94da9ae19c1e GIT binary patch literal 620 zcmV-y0+aoTP)BCfL{N-BPYW9hQLqyf#YWIp4GLO`AB74B zqH8k?!)9;qE+&fNz_B0M$@}i?&b)W6>Byii4KsBualNp)x zAvGYAfKdgh(c#W#3kR28Y<)P}I03*(E8rgN#iAXACyyq4Jb7j@ATQ&?V zsPQ`RUWCAm2!$y?_#{JkG`Lag*W^et|1c?e`R+WTrNOU(eHjl6nKj?$B9(qtQcp94 z#Y*a|=jb%}6>uQ^O+{kWN5W0RG)Gb=J*lR_TTX_!B$`5^<4PFX`K1S3lXwxl)ozqx zBdKP=&w-;HpG`U9szl;T1U@8dESckiep~A0W$G&g`hYXZ?+UO~P Date: Wed, 15 May 2019 20:20:43 +0300 Subject: [PATCH 17/48] Cleanup of documentation issues Task-number: AUTOSUITE-933 Change-Id: I492d7abbf5462b6d922b8ae319902375d23041ee Reviewed-by: Kavindra Palaraja --- doc/src/deployment-server.qdoc | 27 ++++++++++++++++++++++++++- manage.py | 1 + store/utilities.py | 10 +++++----- 3 files changed, 32 insertions(+), 6 deletions(-) mode change 100644 => 100755 manage.py diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index c6b2c45..3c5000d 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -102,7 +102,7 @@ \c{libffi-dev} package. Next, prepare the virtual environment: \code - virtualenv ./venv + virtualenv -p python2.7 ./venv ./venv/bin/pip install -r requirements.txt \endcode @@ -117,6 +117,31 @@ ./manage.py expire-downloads \endcode + \section2 Activate the Python Virtual Environment + + Before you run \c manage.py, source the activation script on the console where you will be using it. + + \code + . ./venv/bin/activate + \endcode + + This activation is done once per session only: + + + \section2 Prepare Your Databases on the Server + + Before you run the server, first, you must prepare the databases. Also, you need to create an administrative user. + + \code + ./manage.py makemigrations + ./manage.py migrate + ./manage.py createsuperuser + \endcode + + Make sure that the server has a user added for Neptune 3 UI's Downloads App. Currently the username and + password are hardcoded in \c apps/com.pelagicore.downloads/stores/ServerConfig.qml:78. + + \section2 Start the Server To start the server, run the following command in your terminal: diff --git a/manage.py b/manage.py old mode 100644 new mode 100755 index ad7604e..af588f4 --- a/manage.py +++ b/manage.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python2.7 ############################################################################# ## ## Copyright (C) 2019 Luxoft Sweden AB diff --git a/store/utilities.py b/store/utilities.py index 8a72661..611579c 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -91,7 +91,7 @@ def downloadPath(): return settings.MEDIA_ROOT + 'downloads/' -def isValidDnsName(dnsName, errorString): +def isValidDnsName(dnsName, errorList): # see also in AM: src/common-lib/utilities.cpp / isValidDnsName() try: @@ -106,7 +106,7 @@ def isValidDnsName(dnsName, errorString): labels = dnsName.split('.') if len(labels) < 3: - raise Exception('wrong format - needs to consist of at least three parts separated by .') + raise Exception('wrong format - needs to be in reverse-DNS notation and consist of at least three parts separated by .') # standard domain name requirements from the RFCs 1035 and 1123 @@ -125,7 +125,7 @@ def isValidDnsName(dnsName, errorString): return True except Exception as error: - errorString = str(error) + errorList[0] = str(error) return False @@ -353,9 +353,9 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): if pkgdata['header']['applicationId'] != pkgdata['info']['id']: raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header']['applicationId'], pkgdata['info']['id'])) - error = '' + error = [''] if not isValidDnsName(pkgdata['info']['id'], error): - raise Exception('invalid id: %s' % error) + raise Exception('invalid id: %s' % error[0]) if pkgdata['header']['diskSpaceUsed'] <= 0: raise Exception('the diskSpaceUsed field in --PACKAGE-HEADER-- is not > 0, but %d' % pkgdata['header']['diskSpaceUsed']) From a3961fb175c5a2a98b75743f6e15e8c06adccdcc Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 9 Jul 2019 18:50:33 +0300 Subject: [PATCH 18/48] Documentation on how to run the appstore behind a reverse proxy Change-Id: I260d87ac94d69be69437392b0c53912383cd00d6 Fixes: AUTOSUITE-501 Reviewed-by: Kavindra Palaraja --- .../deployment-server-http-server-setup.qdoc | 128 ++++++++++++++++++ 1 file changed, 128 insertions(+) create mode 100644 doc/src/deployment-server-http-server-setup.qdoc diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc new file mode 100644 index 0000000..677ab80 --- /dev/null +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -0,0 +1,128 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-http-server-setup.html + \previouspage Qt Automotive Suite Deployment Server + \contentspage Qt Automotive Suite + + + \title Set up a Production server with Apache, Lighttpd or Nginx + +The Deployment Server can be set up in combination with a regular web server: Apache, Lighttpd, or + Nginx. This web server can be used to reduce the number of open ports or to add a layer of SSL + encryption. + +In the Django version that is used for the deployment server, there are two possible setups for + running with another web server: +\list + \li reverse proxy setup - which we describe here + \li WSGI setup - for brevity, this setup is left out of scope, as it is similar in functionality + to reverse proxy setup +\endlist + +\section1 Reverse Proxy Setup + +For all web servers, the preliminary setup is the same. First, the deployment server should be set + up in the same way as for standalone use. If the server will be used in a subdirectory of the + web server (like http://hostname.domain/url_prefix/ ), then this prefix should be set up in + \e{appstore/settings.py}, in a \c{URL_PREFIX} variable. + +In this example, this variable is set to 'url_prefix'; note the lack of slashes. Also, if + \c{URL_PREFIX} is set to a non-empty value, \c{STATIC_URL} should be changed too, to start from + set prefix. So \c{STATIC_URL} would be \c{'//static/'} instead of \c{'/static/'} + +After that, the server should be set up to run as a service, on any free port. +This is done by running: +\code + ./venv/bin/python manage.py runserver 127.0.0.1: +\endcode +Assuming that the Python virtual environment was set up in the project directory, under the + \e{venv} subdirectory. + +On Linix and Unix systems, to run as a non-root user, you must use a port number larger than 1024. + +Once the service is running, set up the web server's configuration. + +\section2 Apache2 Setup + +First, enable \b{mod_proxy} and \b{mod_proxy_http} modules in Apache, by running +\c{a2enmod proxy proxy_http}. + +Then insert this code snippet into the VirtualHost section of your configuration file: + +\code + /'> + ProxyPass "http://127.0.0.1://" + ProxyPassReverse "http://127.0.0.1://" + +\endcode + +If necessary, add configuration section to set up access permissions to this URL. + +After setting this up, reload web server's configuration. + +\section2 Lighttpd Setup + +For Lighttpd, first, enable mod_proxy, by running \c{lighttpd-enable-mod proxy} as root. +Alternatively, add the following line in your \e{/etc/lighttpd/lighttpd.conf} file: +\code +server.modules += ( "mod_proxy" ) +\endcode + +Next, add the following code snippet: +\code +$HTTP["url"] =~ "^//" { + proxy.balance = "hash" + proxy.server = ( "" => ( ( "host" => "127.0.0.1", "port" => "" ), + ) ) +} + +\endcode + +Where \e{} is the same as in \e{settings.py} file, and \e{} is the port on which the +deployment server instance is running. + +Finally, reload the Lighttpd server's configuration. + +\section2 Nginx Setup + +For Nginx, setup consists of adding a code snippet into server configuration, inside of a +\c{server { } } statement and restarting the server. + +The code that needs to be added is: +\code + location // { + proxy_pass http://127.0.0.1://; + } +\endcode + +Where \e{} is the same as in \e{settings.py} file, and \e{} is the port on which the +deployment server instance is running. + +*/ From 0bc3754bd405851661a3912df7ad1afa4db23026 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Thu, 17 Oct 2019 18:06:14 +0300 Subject: [PATCH 19/48] Remove conflict when uploading packages with same architecture but different tags This fix allows for uploading multiple copies of one application with same architecture and different tag sets. Currently check is only done for 'tags are the same', but not for 'tags match'. Change-Id: I5205db092f7edbc77f696a8d878cc52298f72e0f Reviewed-by: Egor Nemtsev --- store/admin.py | 7 ++++--- store/api.py | 6 +++--- store/models.py | 12 ++++++------ store/utilities.py | 16 ++++++++++------ 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/store/admin.py b/store/admin.py index 61d5ff0..491ed8f 100644 --- a/store/admin.py +++ b/store/admin.py @@ -118,6 +118,7 @@ def clean(self): self.appId = pkgdata['info']['id'] self.name = pkgdata['storeName'] self.architecture = pkgdata['architecture'] + self.tags = makeTagList(pkgdata) # check if this really is an update if hasattr(self, 'instance') and self.instance.appid: @@ -127,13 +128,13 @@ def clean(self): raise forms.ValidationError(_('Validation error: an update cannot change the application architecture from %s to %s' % (self.instance.architecture, self.architecture))) else: try: - if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture): - raise forms.ValidationError(_('Validation error: another application with id %s and architecture %s already exists' % (str(self.appId), str(self.architecture)))) + if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture, tags__exact = self.tags): + raise forms.ValidationError(_('Validation error: another application with id %s , tags %s and architecture %s already exists' % (str(self.appId), str(self.tags), str(self.architecture)))) except App.DoesNotExist: pass # write icon into file to serve statically - success, error = writeTempIcon(self.appId, self.architecture, pkgdata['icon']) + success, error = writeTempIcon(self.appId, self.architecture, self.tags, pkgdata['icon']) if not success: raise forms.ValidationError(_(error)) diff --git a/store/api.py b/store/api.py index 1c26944..de975dd 100644 --- a/store/api.py +++ b/store/api.py @@ -238,7 +238,7 @@ def appIcon(request): try: app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture') app = app.last() - with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: + with open(iconPath(app.appid,app.architecture,app.tags), 'rb') as iconPng: response = HttpResponse(content_type = 'image/png') response.write(iconPng.read()) return response @@ -262,11 +262,11 @@ def appPurchase(request): app = App.objects.filter(appid__exact = getRequestDictionary(request)['id'], architecture__in=archlist).order_by('architecture') app = app.last() - fromFilePath = packagePath(app.appid, app.architecture) + fromFilePath = packagePath(app.appid, app.architecture, app.tags) # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) - toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(deviceId) + '.appkg' + toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + '.appkg' toPath = downloadPath() if not os.path.exists(toPath): os.makedirs(toPath) diff --git a/store/models.py b/store/models.py index 08bebc3..aa8bf25 100644 --- a/store/models.py +++ b/store/models.py @@ -86,7 +86,7 @@ def __unicode__(self): def content_file_name(instance, filename): - return packagePath(instance.appid, instance.architecture) + return packagePath(instance.appid, instance.architecture, instance.tags) class App(models.Model): appid = models.CharField(max_length = 200) @@ -111,7 +111,7 @@ def __unicode__(self): def save(self, *args, **kwargs): try: - this = App.objects.get(appid=self.appid,architecture=self.architecture) + this = App.objects.get(appid=self.appid,architecture=self.architecture,tags=self.tags) #FIXME: This should be 'tags match, not exact same tags' if this.file != self.file: this.file.delete(save=False) except: @@ -124,14 +124,14 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri name = pkgdata['storeName'] architecture = pkgdata['architecture'] tags = makeTagList(pkgdata) - success, error = writeTempIcon(appId, architecture, pkgdata['icon']) + success, error = writeTempIcon(appId, architecture, tags, pkgdata['icon']) if not success: raise Exception(error) exists = False app = None try: - app = App.objects.get(appid__exact=appId, architecture__exact=architecture) + app = App.objects.get(appid__exact=appId, architecture__exact=architecture, tags__exact=tags) exists = True except App.DoesNotExist: pass @@ -145,12 +145,12 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri app.description = description app.briefDescription = shortdescription app.architecture = architecture - app.file.save(packagePath(appId, architecture), pkgfile) + app.file.save(packagePath(appId, architecture, tags), pkgfile) app.save() else: app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, category=category, appid=appId, briefDescription=shortdescription, description=description, architecture=architecture) - app.file.save(packagePath(appId, architecture), pkgfile) + app.file.save(packagePath(appId, architecture, tags), pkgfile) app.save() diff --git a/store/utilities.py b/store/utilities.py index 611579c..cec403a 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -63,23 +63,27 @@ def getRequestDictionary(request): else: return request.GET -def packagePath(appId = None, architecture = None): +def packagePath(appId = None, architecture = None, tags = None): path = settings.MEDIA_ROOT + 'packages/' + if tags is None: + tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') + return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') return path -def iconPath(appId = None, architecture = None): +def iconPath(appId = None, architecture = None, tags = None): path = settings.MEDIA_ROOT + 'icons/' + if tags is None: + tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') + '.png' + return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') + '.png' return path -def writeTempIcon(appId, architecture, icon): +def writeTempIcon(appId, architecture, tags, icon): try: if not os.path.exists(iconPath()): os.makedirs(iconPath()) - tempicon = open(iconPath(appId, architecture), 'w') + tempicon = open(iconPath(appId, architecture, tags), 'w') tempicon.write(icon) tempicon.flush() tempicon.close() From 97a35668762052b6a667b3b3b823367e9f38aef1 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 17 Jan 2020 17:55:01 +0300 Subject: [PATCH 20/48] Django version update in response to CVE-2019-19844 Change-Id: I0c4484f5f4f3184bd570776cb318dd534f46817c Reviewed-by: Nikolay Zamotaev --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 41de88c..c605671 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pkg-resources==0.0.0 PyYAML -django==1.11 +django==1.11.27 django-common django-ordered-model==2.1 pyOpenSSL From b7c57b7f83c5dffe54642811ca921d57a4f91ef3 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 17 Jan 2020 17:55:01 +0300 Subject: [PATCH 21/48] Django version update in response to CVE-2019-19844 Change-Id: Ib2f34786600a7ecdefba068261106e545996b752 Reviewed-by: Nikolay Zamotaev --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 41de88c..c605671 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ pkg-resources==0.0.0 PyYAML -django==1.11 +django==1.11.27 django-common django-ordered-model==2.1 pyOpenSSL From a095cc3470c473f123c032f3fcfc599e006dad40 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 31 Jan 2020 19:56:06 +0300 Subject: [PATCH 22/48] Support for reporting wrong architecture in /hello status Task-number: AUTOSUITE-1438 Change-Id: Id02885b829c21641da456f99e054541b09cf964e Reviewed-by: Egor Nemtsev --- doc/src/deployment-server-reference.qdoc | 7 +++++-- store/api.py | 5 ++++- store/osandarch.py | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index c49c3d2..b81bd7f 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -78,7 +78,7 @@ \li Value \li Description \row - \li {1,5} status + \li {1,6} status \li ok \li Successful \row @@ -90,6 +90,9 @@ \row \li incompatible-version \li The API version you are using is not compatible. + \row + \li incompatible-architecture + \li The architecture parameter was malformed, or server was unable to parse it. \row \li malformed-tag \li The tag format is incorrect, may not be alphanumeric, or could @@ -444,4 +447,4 @@ seconds. \endlist -*/ \ No newline at end of file +*/ diff --git a/store/api.py b/store/api.py index de975dd..09b6550 100644 --- a/store/api.py +++ b/store/api.py @@ -65,7 +65,10 @@ def hello(request): break request.session[j] = str(versionmap) if 'architecture' in getRequestDictionary(request): - request.session['architecture'] = normalizeArch(getRequestDictionary(request)['architecture']) + arch = normalizeArch(getRequestDictionary(request)['architecture']) + if arch == "": + status = 'incompatible-architecture' + request.session['architecture'] = arch else: request.session['architecture'] = '' return JsonResponse({'status': status}) diff --git a/store/osandarch.py b/store/osandarch.py index 9a2db12..0dde800 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -138,6 +138,8 @@ def normalizeArch(inputArch): """ parts = inputArch.split('-') + if len(parts) <4: + return "" # Invalid format #Drop anything non-numeric from word_size field parts[2]=re.sub(r"\D", "", parts[2]) #Transform kernelType into binary format From 086fc8c681f63ecf099c8d167f050bf4177235ed Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 28 Jan 2020 17:12:38 +0300 Subject: [PATCH 23/48] App Icon request now uses architecture parameter along with session parameters Task-number: AUTOSUITE-1424 Change-Id: I76ddb3e5e42eeb280f978804ea28d200a838958e Reviewed-by: Egor Nemtsev --- doc/src/deployment-server-reference.qdoc | 6 ++++++ store/api.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index b81bd7f..1ade88f 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -227,6 +227,12 @@ \row \li id \li The app ID. + \row + \li architecture + \li An optional parameter used to filter icons by architecture. Overrides architecture + specified in session. Receives the CPU architecture. If architecture was not specified + either in session or in this parameter, only icons showing \e{All} architecture + are listed. \endtable Returns a PNG image if the app exists; an HTTP 404 error otherwise. diff --git a/store/api.py b/store/api.py index 09b6550..04f6daf 100644 --- a/store/api.py +++ b/store/api.py @@ -235,9 +235,12 @@ def appDescription(request): def appIcon(request): archlist = ['All', ] - if 'architecture' in request.session: + dictionary = getRequestDictionary(request) + if 'architecture' in dictionary: + archlist.append(normalizeArch(dictionary['architecture'])) + elif 'architecture' in request.session: archlist.append(request.session['architecture']) - appId = getRequestDictionary(request)['id'] + appId = dictionary['id'] try: app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture') app = app.last() From a7bb59eaaefa59f17e0c29fc8371b84cf24f0aa5 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 4 Feb 2020 18:07:38 +0300 Subject: [PATCH 24/48] File path cleanup in 'missing file' error message Change-Id: I34138d2b288f577b726a7b79178e7245ec02671f Fixes: AUTOSUITE-1451 Reviewed-by: Egor Nemtsev --- store/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/store/api.py b/store/api.py index 04f6daf..003d467 100644 --- a/store/api.py +++ b/store/api.py @@ -282,7 +282,13 @@ def appPurchase(request): pkgdata = parsePackageMetadata(package) addSignatureToPackage(fromFilePath, toPath + toFile, pkgdata['rawDigest'], deviceId) else: - shutil.copyfile(fromFilePath, toPath + toFile) + try: + shutil.copyfile(fromFilePath, toPath + toFile) + except Exception as error: + if type(error) == IOError: + raise IOError(error.args[0],error.args[1], os.path.basename(fromFilePath)) + else: + raise error if settings.URL_PREFIX != '': downloadUri = '/' + settings.URL_PREFIX + '/app/download/' + toFile From f605d6601bf52631e27e97454a74fdaa64a05a83 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 4 Feb 2020 17:55:31 +0300 Subject: [PATCH 25/48] Fix for django bug, where : and , symbols are not properly handled When saving package files, ':' and ',' symbols were omitted from the filename, even when they were passed properly. This code substitutes them for their hexadecimal value. Also this code changes download URLs to a safer version (names are hashed with sha256, so they are unguessable). This only happens when DEBUG is set to False Change-Id: Iba2b52c6aef0b416ac7a2c276aa0ae72904be70d Fixes: AUTOSUITE-1450 Reviewed-by: Egor Nemtsev --- store/api.py | 6 +++++- store/utilities.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/store/api.py b/store/api.py index 003d467..3384df1 100644 --- a/store/api.py +++ b/store/api.py @@ -32,6 +32,7 @@ import os import shutil +import hashlib from django.conf import settings from django.db.models import Q, Count @@ -272,7 +273,10 @@ def appPurchase(request): # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) - toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + '.appkg' + toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + if not settings.DEBUG: + toFile = hashlib.sha256(toFile).hexdigest() + toFile += '.appkg' toPath = downloadPath() if not os.path.exists(toPath): os.makedirs(toPath) diff --git a/store/utilities.py b/store/utilities.py index cec403a..02faf4a 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -68,7 +68,7 @@ def packagePath(appId = None, architecture = None, tags = None): if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') + path = path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_').replace(':','x3A').replace(',','x2C') return path def iconPath(appId = None, architecture = None, tags = None): @@ -76,7 +76,7 @@ def iconPath(appId = None, architecture = None, tags = None): if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') + '.png' + return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_').replace(':','x3A').replace(',','x2C') + '.png' return path def writeTempIcon(appId, architecture, tags, icon): From 647a3b6e0dbfce90ac24712c19ae0a4bfa596103 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Thu, 17 Oct 2019 18:06:14 +0300 Subject: [PATCH 26/48] Remove conflict when uploading packages with same architecture but different tags This fix allows for uploading multiple copies of one application with same architecture and different tag sets. Currently check is only done for 'tags are the same', but not for 'tags match'. Change-Id: I5205db092f7edbc77f696a8d878cc52298f72e0f Reviewed-by: Egor Nemtsev (cherry picked from commit 0bc3754bd405851661a3912df7ad1afa4db23026) Reviewed-by: Nikolay Zamotaev --- store/admin.py | 7 ++++--- store/api.py | 6 +++--- store/models.py | 12 ++++++------ store/utilities.py | 16 ++++++++++------ 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/store/admin.py b/store/admin.py index 61d5ff0..491ed8f 100644 --- a/store/admin.py +++ b/store/admin.py @@ -118,6 +118,7 @@ def clean(self): self.appId = pkgdata['info']['id'] self.name = pkgdata['storeName'] self.architecture = pkgdata['architecture'] + self.tags = makeTagList(pkgdata) # check if this really is an update if hasattr(self, 'instance') and self.instance.appid: @@ -127,13 +128,13 @@ def clean(self): raise forms.ValidationError(_('Validation error: an update cannot change the application architecture from %s to %s' % (self.instance.architecture, self.architecture))) else: try: - if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture): - raise forms.ValidationError(_('Validation error: another application with id %s and architecture %s already exists' % (str(self.appId), str(self.architecture)))) + if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture, tags__exact = self.tags): + raise forms.ValidationError(_('Validation error: another application with id %s , tags %s and architecture %s already exists' % (str(self.appId), str(self.tags), str(self.architecture)))) except App.DoesNotExist: pass # write icon into file to serve statically - success, error = writeTempIcon(self.appId, self.architecture, pkgdata['icon']) + success, error = writeTempIcon(self.appId, self.architecture, self.tags, pkgdata['icon']) if not success: raise forms.ValidationError(_(error)) diff --git a/store/api.py b/store/api.py index 1c26944..de975dd 100644 --- a/store/api.py +++ b/store/api.py @@ -238,7 +238,7 @@ def appIcon(request): try: app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture') app = app.last() - with open(iconPath(app.appid,app.architecture), 'rb') as iconPng: + with open(iconPath(app.appid,app.architecture,app.tags), 'rb') as iconPng: response = HttpResponse(content_type = 'image/png') response.write(iconPng.read()) return response @@ -262,11 +262,11 @@ def appPurchase(request): app = App.objects.filter(appid__exact = getRequestDictionary(request)['id'], architecture__in=archlist).order_by('architecture') app = app.last() - fromFilePath = packagePath(app.appid, app.architecture) + fromFilePath = packagePath(app.appid, app.architecture, app.tags) # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) - toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(deviceId) + '.appkg' + toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + '.appkg' toPath = downloadPath() if not os.path.exists(toPath): os.makedirs(toPath) diff --git a/store/models.py b/store/models.py index 08bebc3..aa8bf25 100644 --- a/store/models.py +++ b/store/models.py @@ -86,7 +86,7 @@ def __unicode__(self): def content_file_name(instance, filename): - return packagePath(instance.appid, instance.architecture) + return packagePath(instance.appid, instance.architecture, instance.tags) class App(models.Model): appid = models.CharField(max_length = 200) @@ -111,7 +111,7 @@ def __unicode__(self): def save(self, *args, **kwargs): try: - this = App.objects.get(appid=self.appid,architecture=self.architecture) + this = App.objects.get(appid=self.appid,architecture=self.architecture,tags=self.tags) #FIXME: This should be 'tags match, not exact same tags' if this.file != self.file: this.file.delete(save=False) except: @@ -124,14 +124,14 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri name = pkgdata['storeName'] architecture = pkgdata['architecture'] tags = makeTagList(pkgdata) - success, error = writeTempIcon(appId, architecture, pkgdata['icon']) + success, error = writeTempIcon(appId, architecture, tags, pkgdata['icon']) if not success: raise Exception(error) exists = False app = None try: - app = App.objects.get(appid__exact=appId, architecture__exact=architecture) + app = App.objects.get(appid__exact=appId, architecture__exact=architecture, tags__exact=tags) exists = True except App.DoesNotExist: pass @@ -145,12 +145,12 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri app.description = description app.briefDescription = shortdescription app.architecture = architecture - app.file.save(packagePath(appId, architecture), pkgfile) + app.file.save(packagePath(appId, architecture, tags), pkgfile) app.save() else: app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, category=category, appid=appId, briefDescription=shortdescription, description=description, architecture=architecture) - app.file.save(packagePath(appId, architecture), pkgfile) + app.file.save(packagePath(appId, architecture, tags), pkgfile) app.save() diff --git a/store/utilities.py b/store/utilities.py index 611579c..cec403a 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -63,23 +63,27 @@ def getRequestDictionary(request): else: return request.GET -def packagePath(appId = None, architecture = None): +def packagePath(appId = None, architecture = None, tags = None): path = settings.MEDIA_ROOT + 'packages/' + if tags is None: + tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') + return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') return path -def iconPath(appId = None, architecture = None): +def iconPath(appId = None, architecture = None, tags = None): path = settings.MEDIA_ROOT + 'icons/' + if tags is None: + tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture]).replace('/','_').replace('\\','_') + '.png' + return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') + '.png' return path -def writeTempIcon(appId, architecture, icon): +def writeTempIcon(appId, architecture, tags, icon): try: if not os.path.exists(iconPath()): os.makedirs(iconPath()) - tempicon = open(iconPath(appId, architecture), 'w') + tempicon = open(iconPath(appId, architecture, tags), 'w') tempicon.write(icon) tempicon.flush() tempicon.close() From 9c45aeb3aafe236093b96021d5c6b170f9c4b126 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 31 Jan 2020 19:56:06 +0300 Subject: [PATCH 27/48] Support for reporting wrong architecture in /hello status Task-number: AUTOSUITE-1438 Change-Id: Id02885b829c21641da456f99e054541b09cf964e Reviewed-by: Egor Nemtsev (cherry picked from commit a095cc3470c473f123c032f3fcfc599e006dad40) Reviewed-by: Nikolay Zamotaev --- doc/src/deployment-server-reference.qdoc | 7 +++++-- store/api.py | 5 ++++- store/osandarch.py | 2 ++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index c49c3d2..b81bd7f 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -78,7 +78,7 @@ \li Value \li Description \row - \li {1,5} status + \li {1,6} status \li ok \li Successful \row @@ -90,6 +90,9 @@ \row \li incompatible-version \li The API version you are using is not compatible. + \row + \li incompatible-architecture + \li The architecture parameter was malformed, or server was unable to parse it. \row \li malformed-tag \li The tag format is incorrect, may not be alphanumeric, or could @@ -444,4 +447,4 @@ seconds. \endlist -*/ \ No newline at end of file +*/ diff --git a/store/api.py b/store/api.py index de975dd..09b6550 100644 --- a/store/api.py +++ b/store/api.py @@ -65,7 +65,10 @@ def hello(request): break request.session[j] = str(versionmap) if 'architecture' in getRequestDictionary(request): - request.session['architecture'] = normalizeArch(getRequestDictionary(request)['architecture']) + arch = normalizeArch(getRequestDictionary(request)['architecture']) + if arch == "": + status = 'incompatible-architecture' + request.session['architecture'] = arch else: request.session['architecture'] = '' return JsonResponse({'status': status}) diff --git a/store/osandarch.py b/store/osandarch.py index 9a2db12..0dde800 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -138,6 +138,8 @@ def normalizeArch(inputArch): """ parts = inputArch.split('-') + if len(parts) <4: + return "" # Invalid format #Drop anything non-numeric from word_size field parts[2]=re.sub(r"\D", "", parts[2]) #Transform kernelType into binary format From d3199f15e5919dbaefbb3e1fab1520c67f595193 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 28 Jan 2020 17:12:38 +0300 Subject: [PATCH 28/48] App Icon request now uses architecture parameter along with session parameters Task-number: AUTOSUITE-1424 Change-Id: I76ddb3e5e42eeb280f978804ea28d200a838958e Reviewed-by: Egor Nemtsev (cherry picked from commit 086fc8c681f63ecf099c8d167f050bf4177235ed) Reviewed-by: Nikolay Zamotaev --- doc/src/deployment-server-reference.qdoc | 6 ++++++ store/api.py | 7 +++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index b81bd7f..1ade88f 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -227,6 +227,12 @@ \row \li id \li The app ID. + \row + \li architecture + \li An optional parameter used to filter icons by architecture. Overrides architecture + specified in session. Receives the CPU architecture. If architecture was not specified + either in session or in this parameter, only icons showing \e{All} architecture + are listed. \endtable Returns a PNG image if the app exists; an HTTP 404 error otherwise. diff --git a/store/api.py b/store/api.py index 09b6550..04f6daf 100644 --- a/store/api.py +++ b/store/api.py @@ -235,9 +235,12 @@ def appDescription(request): def appIcon(request): archlist = ['All', ] - if 'architecture' in request.session: + dictionary = getRequestDictionary(request) + if 'architecture' in dictionary: + archlist.append(normalizeArch(dictionary['architecture'])) + elif 'architecture' in request.session: archlist.append(request.session['architecture']) - appId = getRequestDictionary(request)['id'] + appId = dictionary['id'] try: app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture') app = app.last() From df22adb35d188afe7eb7424ba9c3c055e55c3566 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 4 Feb 2020 18:07:38 +0300 Subject: [PATCH 29/48] File path cleanup in 'missing file' error message Change-Id: I34138d2b288f577b726a7b79178e7245ec02671f Fixes: AUTOSUITE-1451 Reviewed-by: Egor Nemtsev (cherry picked from commit a7bb59eaaefa59f17e0c29fc8371b84cf24f0aa5) Reviewed-by: Nikolay Zamotaev --- store/api.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/store/api.py b/store/api.py index 04f6daf..003d467 100644 --- a/store/api.py +++ b/store/api.py @@ -282,7 +282,13 @@ def appPurchase(request): pkgdata = parsePackageMetadata(package) addSignatureToPackage(fromFilePath, toPath + toFile, pkgdata['rawDigest'], deviceId) else: - shutil.copyfile(fromFilePath, toPath + toFile) + try: + shutil.copyfile(fromFilePath, toPath + toFile) + except Exception as error: + if type(error) == IOError: + raise IOError(error.args[0],error.args[1], os.path.basename(fromFilePath)) + else: + raise error if settings.URL_PREFIX != '': downloadUri = '/' + settings.URL_PREFIX + '/app/download/' + toFile From 811ff184e3cd925791cd3b68eb8e744db38c9653 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Tue, 4 Feb 2020 17:55:31 +0300 Subject: [PATCH 30/48] Fix for django bug, where : and , symbols are not properly handled When saving package files, ':' and ',' symbols were omitted from the filename, even when they were passed properly. This code substitutes them for their hexadecimal value. Also this code changes download URLs to a safer version (names are hashed with sha256, so they are unguessable). This only happens when DEBUG is set to False Change-Id: Iba2b52c6aef0b416ac7a2c276aa0ae72904be70d Fixes: AUTOSUITE-1450 Reviewed-by: Egor Nemtsev (cherry picked from commit f605d6601bf52631e27e97454a74fdaa64a05a83) Reviewed-by: Nikolay Zamotaev --- store/api.py | 6 +++++- store/utilities.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/store/api.py b/store/api.py index 003d467..3384df1 100644 --- a/store/api.py +++ b/store/api.py @@ -32,6 +32,7 @@ import os import shutil +import hashlib from django.conf import settings from django.db.models import Q, Count @@ -272,7 +273,10 @@ def appPurchase(request): # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) - toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + '.appkg' + toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + if not settings.DEBUG: + toFile = hashlib.sha256(toFile).hexdigest() + toFile += '.appkg' toPath = downloadPath() if not os.path.exists(toPath): os.makedirs(toPath) diff --git a/store/utilities.py b/store/utilities.py index cec403a..02faf4a 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -68,7 +68,7 @@ def packagePath(appId = None, architecture = None, tags = None): if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') + path = path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_').replace(':','x3A').replace(',','x2C') return path def iconPath(appId = None, architecture = None, tags = None): @@ -76,7 +76,7 @@ def iconPath(appId = None, architecture = None, tags = None): if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_') + '.png' + return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_').replace(':','x3A').replace(',','x2C') + '.png' return path def writeTempIcon(appId, architecture, tags, icon): From 2c6654413f952eecb3ce81579d27908438f3a507 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 29 Jan 2020 18:17:51 +0300 Subject: [PATCH 31/48] Documentation and settings update for use with static files served separately Task-number: AUTOSUITE-1429 Change-Id: Ib670d8c41f617808db742289a4ae4ab1d77a1919 Reviewed-by: Egor Nemtsev --- .gitignore | 1 + appstore/settings.py | 1 + doc/src/deployment-server-http-server-setup.qdoc | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/.gitignore b/.gitignore index 1baae07..306e8fe 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ media/ certificates/ .idea/* venv/* +static/* diff --git a/appstore/settings.py b/appstore/settings.py index 8e67698..5babd8d 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -149,6 +149,7 @@ # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" MEDIA_ROOT = os.path.join(BASE_DIR, 'media/') +STATIC_ROOT = os.path.join(BASE_DIR, 'static') # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc index 677ab80..a293931 100644 --- a/doc/src/deployment-server-http-server-setup.qdoc +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -125,4 +125,18 @@ The code that needs to be added is: Where \e{} is the same as in \e{settings.py} file, and \e{} is the port on which the deployment server instance is running. +\section1 Serving static files outside of django app + +It is possible to speed up working of the admin pages of deployment server by serving static files +separately. In order to achieve that, webserver should be configured to serve \e{static/} subdirectory of +deployment server installation as \e{static/} subdirectory of the instance. (If deployment server is +hosted as \e{http://deployment.server.name/}, then \e{static/} directory in the sources should be redirected +to \e{http://deployment.server.name/static/}, while bypassing the django app.) + +After configuring that, static files collection should be performed with this command: + +\code + ./venv/bin/python manage.py collect static +\endcode + */ From 25b6532ee24b51300a794589acf0058c9f8f5d60 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 29 Jan 2020 18:17:51 +0300 Subject: [PATCH 32/48] Documentation and settings update for use with static files served separately Task-number: AUTOSUITE-1429 Change-Id: Ib670d8c41f617808db742289a4ae4ab1d77a1919 Reviewed-by: Egor Nemtsev (cherry picked from commit 2c6654413f952eecb3ce81579d27908438f3a507) Reviewed-by: Nikolay Zamotaev --- .gitignore | 1 + appstore/settings.py | 1 + doc/src/deployment-server-http-server-setup.qdoc | 14 ++++++++++++++ 3 files changed, 16 insertions(+) diff --git a/.gitignore b/.gitignore index 1baae07..306e8fe 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ media/ certificates/ .idea/* venv/* +static/* diff --git a/appstore/settings.py b/appstore/settings.py index 8e67698..5babd8d 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -149,6 +149,7 @@ # Absolute path to the directory that holds media. # Example: "/home/media/media.lawrence.com/" MEDIA_ROOT = os.path.join(BASE_DIR, 'media/') +STATIC_ROOT = os.path.join(BASE_DIR, 'static') # URL that handles the media served from MEDIA_ROOT. Make sure to use a # trailing slash if there is a path component (optional in other cases). diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc index 677ab80..a293931 100644 --- a/doc/src/deployment-server-http-server-setup.qdoc +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -125,4 +125,18 @@ The code that needs to be added is: Where \e{} is the same as in \e{settings.py} file, and \e{} is the port on which the deployment server instance is running. +\section1 Serving static files outside of django app + +It is possible to speed up working of the admin pages of deployment server by serving static files +separately. In order to achieve that, webserver should be configured to serve \e{static/} subdirectory of +deployment server installation as \e{static/} subdirectory of the instance. (If deployment server is +hosted as \e{http://deployment.server.name/}, then \e{static/} directory in the sources should be redirected +to \e{http://deployment.server.name/static/}, while bypassing the django app.) + +After configuring that, static files collection should be performed with this command: + +\code + ./venv/bin/python manage.py collect static +\endcode + */ From 27398cfb9ef0decab97c7f84e64a1d1a9613fd18 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 7 Feb 2020 19:56:36 +0300 Subject: [PATCH 33/48] Fix for Qt AppMan 5.14 support Qt 5.14 brough new package format. This change brings backward-compatible support for new and old packages in the same deployment server. Supported formats are differentiated by version parameter in /hello API call (version 1 - only old packages, version 2 - new and old packages) Task-number: AUTOSUITE-1356 Change-Id: Ifcd65f162dbadf069f2bb4f506482bbda6a2984e Reviewed-by: Egor Nemtsev --- .gitignore | 1 + appstore/settings.py | 4 +- store/admin.py | 15 ++-- store/api.py | 37 +++++++--- .../management/commands/store-sign-package.py | 5 +- store/migrations/0001_initial.py | 5 +- store/models.py | 17 +++-- store/utilities.py | 74 +++++++++++++------ 8 files changed, 105 insertions(+), 53 deletions(-) diff --git a/.gitignore b/.gitignore index 306e8fe..cd45596 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.pyc db.sqlite3 media/ +static/ certificates/ .idea/* venv/* diff --git a/appstore/settings.py b/appstore/settings.py index 5babd8d..2224284 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -42,7 +42,9 @@ APPSTORE_MAINTENANCE = False APPSTORE_PLATFORM_ID = 'NEPTUNE3' -APPSTORE_PLATFORM_VERSION = 1 +APPSTORE_PLATFORM_VERSION = 2 # Maximum supported platform version: + # version 1 - only old package format + # version 2 - old and new package formats APPSTORE_DOWNLOAD_EXPIRY = 10 # in minutes APPSTORE_BIND_TO_DEVICE_ID = True # unique downloads for each device APPSTORE_NO_SECURITY = True # ignore developer signatures and do not generate store signatures diff --git a/store/admin.py b/store/admin.py index 491ed8f..2252af1 100644 --- a/store/admin.py +++ b/store/admin.py @@ -1,6 +1,6 @@ ############################################################################# ## -## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2020 Luxoft Sweden AB ## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## @@ -31,17 +31,18 @@ ############################################################################# import os +import StringIO +from PIL import Image, ImageChops from django import forms from django.contrib import admin from django.utils.translation import ugettext as _ from django.utils.translation import ugettext_lazy -from ordered_model.admin import OrderedModelAdmin from django.core.files.uploadedfile import InMemoryUploadedFile +from ordered_model.admin import OrderedModelAdmin from store.models import * from utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList -import StringIO class CategoryAdminForm(forms.ModelForm): class Meta: @@ -65,7 +66,7 @@ def clean(self): im = im.convert('LA') else: # No conversion, icons are uploaded as-is, only scaling is used. - im = Image.open(cleared_data['icon']) + im = Image.open(cleaned_data['icon']) size = (settings.ICON_SIZE_X,settings.ICON_SIZE_Y,) im.thumbnail(size, Image.ANTIALIAS) imagefile = StringIO.StringIO() @@ -99,7 +100,7 @@ def icon_image(self, obj): class AppAdminForm(forms.ModelForm): class Meta: - exclude = ["appid", "name", "tags", "architecture", 'version'] + exclude = ["appid", "name", "tags", "architecture", 'version', 'pkgformat'] appId = "" name = "" @@ -109,7 +110,6 @@ def clean(self): file = cleaned_data.get('file') # validate package - pkgdata = None try: pkgdata = parseAndValidatePackageMetadata(file) except Exception as error: @@ -149,12 +149,13 @@ def save(self, commit=False): m.file.seek(0) pkgdata = parseAndValidatePackageMetadata(m.file) m.tags = makeTagList(pkgdata) + m.pkgformat = pkgdata['packageFormat']['formatVersion'] return m class AppAdmin(admin.ModelAdmin): form = AppAdminForm - list_display = ('name', 'appid', 'architecture', 'version', 'tags') + list_display = ('name', 'appid', 'architecture', 'version', 'pkgformat', 'tags') def save_model(self, request, obj, form, change): obj.save() diff --git a/store/api.py b/store/api.py index 3384df1..fe8e25d 100644 --- a/store/api.py +++ b/store/api.py @@ -51,27 +51,38 @@ def hello(request): status = 'ok' + dictionary = getRequestDictionary(request) + try: + version = int(dictionary.get("version", "-1")) + if version > 256: #Sanity check against DoS attack (memory exhaustion) + version = -1 + except: + version = -1 + if settings.APPSTORE_MAINTENANCE: status = 'maintenance' - elif getRequestDictionary(request).get("platform", "") != str(settings.APPSTORE_PLATFORM_ID): + elif dictionary.get("platform", "") != str(settings.APPSTORE_PLATFORM_ID): status = 'incompatible-platform' - elif getRequestDictionary(request).get("version", "") != str(settings.APPSTORE_PLATFORM_VERSION): + elif not ((version) > 0 and (version <= settings.APPSTORE_PLATFORM_VERSION)): status = 'incompatible-version' for j in ("require_tag", "conflicts_tag",): - if j in getRequestDictionary(request): #Tags are coma-separated, + if j in dictionary: #Tags are coma-separated, versionmap = SoftwareTagList() if not versionmap.parse(getRequestDictionary(request)[j]): status = 'malformed-tag' break request.session[j] = str(versionmap) - if 'architecture' in getRequestDictionary(request): + + if 'architecture' in dictionary: arch = normalizeArch(getRequestDictionary(request)['architecture']) if arch == "": status = 'incompatible-architecture' request.session['architecture'] = arch else: request.session['architecture'] = '' + + request.session['pkgversions'] = range(1, version + 1) return JsonResponse({'status': status}) @@ -161,10 +172,11 @@ def upload(request): def appList(request): apps = App.objects.all() - if 'filter' in getRequestDictionary(request): - apps = apps.filter(name__contains = getRequestDictionary(request)['filter']) - if 'category_id' in getRequestDictionary(request): - catId = getRequestDictionary(request)['category_id'] + dictionary = getRequestDictionary(request) + if 'filter' in dictionary: + apps = apps.filter(name__contains = dictionary['filter']) + if 'category_id' in dictionary: + catId = dictionary['category_id'] if catId != -1: # All metacategory apps = apps.filter(category__exact = catId) @@ -189,7 +201,13 @@ def appList(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) + + versionlist = [1] + if 'pkgversions' in request.session: + versionlist = request.session['pkgversions'] + apps = apps.filter(architecture__in = archlist) + apps = apps.filter(pkgformat__in = versionlist) # After filtering, there are potential duplicates in list. And we should prefer native applications to pure qml ones # due to speedups offered. @@ -284,7 +302,8 @@ def appPurchase(request): if not settings.APPSTORE_NO_SECURITY: with open(fromFilePath, 'rb') as package: pkgdata = parsePackageMetadata(package) - addSignatureToPackage(fromFilePath, toPath + toFile, pkgdata['rawDigest'], deviceId) + addSignatureToPackage(fromFilePath, toPath + toFile, pkgdata['rawDigest'], deviceId, + pkgdata['packageFormat']['formatVersion']) else: try: shutil.copyfile(fromFilePath, toPath + toFile) diff --git a/store/management/commands/store-sign-package.py b/store/management/commands/store-sign-package.py index b1a42d0..cf51670 100644 --- a/store/management/commands/store-sign-package.py +++ b/store/management/commands/store-sign-package.py @@ -30,8 +30,6 @@ ## ############################################################################# -import sys - from django.core.management.base import BaseCommand, CommandError from django.conf import settings @@ -55,7 +53,8 @@ def handle(self, *args, **options): self.stdout.write(' -> passed validation (internal name: %s)\n' % pkgdata['storeName']) self.stdout.write('Adding signature to package %s' % destinationPackage) - addSignatureToPackage(sourcePackage, destinationPackage, pkgdata['rawDigest'], deviceId) + addSignatureToPackage(sourcePackage, destinationPackage, pkgdata['rawDigest'], + deviceId, pkgdata['packageFormat']['formatVersion']) self.stdout.write(' -> finished') except Exception as error: diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index dca1599..d0e2c04 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- ############################################################################# ## -## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2020 Luxoft Sweden AB ## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## @@ -31,7 +31,7 @@ ## ############################################################################# -# Generated by Django 1.11 on 2019-03-25 14:51 +# Generated by Django 1.11.27 on 2020-02-07 16:50 from __future__ import unicode_literals from django.conf import settings @@ -63,6 +63,7 @@ class Migration(migrations.Migration): ('tags', models.TextField(blank=True)), ('architecture', models.CharField(default=b'All', max_length=20)), ('version', models.CharField(default=b'0.0.0', max_length=20)), + ('pkgformat', models.IntegerField()), ], ), migrations.CreateModel( diff --git a/store/models.py b/store/models.py index aa8bf25..93136af 100644 --- a/store/models.py +++ b/store/models.py @@ -1,6 +1,6 @@ ############################################################################# ## -## Copyright (C) 2019 Luxoft Sweden AB +## Copyright (C) 2020 Luxoft Sweden AB ## Copyright (C) 2018 Pelagicore AG ## Contact: https://www.qt.io/licensing/ ## @@ -31,14 +31,12 @@ ############################################################################# import os -from PIL import Image, ImageChops from django.db import models from ordered_model.models import OrderedModel from django.conf import settings from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage -from django.db.models.fields.files import ImageFieldFile from utilities import packagePath, writeTempIcon, makeTagList @@ -101,13 +99,14 @@ class App(models.Model): tags = models.TextField(blank=True) architecture = models.CharField(max_length=20, default='All') version = models.CharField(max_length=20, default='0.0.0') + pkgformat = models.IntegerField() class Meta: """Makes the group of id and arch - a unique identifier""" unique_together = (('appid', 'architecture', 'tags'),) def __unicode__(self): - return self.name + " [" + " ".join([self.appid,self.version,self.architecture,self.tags]) + "]" + return self.name + " [" + " ".join([self.appid, self.version, self.architecture, self.tags]) + "]" def save(self, *args, **kwargs): try: @@ -123,6 +122,7 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri appId = pkgdata['info']['id'] name = pkgdata['storeName'] architecture = pkgdata['architecture'] + pkgformat = pkgdata['packageFormat']['formatVersion'] tags = makeTagList(pkgdata) success, error = writeTempIcon(appId, architecture, tags, pkgdata['icon']) if not success: @@ -145,12 +145,13 @@ def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescri app.description = description app.briefDescription = shortdescription app.architecture = architecture + app.pkgformat = pkgformat app.file.save(packagePath(appId, architecture, tags), pkgfile) app.save() else: - app, created = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, - category=category, appid=appId, - briefDescription=shortdescription, description=description, - architecture=architecture) + app, _ = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, + category=category, appid=appId, + briefDescription=shortdescription, description=description, + pkgformat = pkgformat, architecture=architecture) app.file.save(packagePath(appId, architecture, tags), pkgfile) app.save() diff --git a/store/utilities.py b/store/utilities.py index 02faf4a..1205f1b 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -30,21 +30,19 @@ ## ############################################################################# -import tarfile -import hashlib -import hmac -import yaml -import sys import tarfile import tempfile import base64 import os +import hashlib +import hmac +import yaml import magic -from M2Crypto import SMIME, BIO, X509 +from django.conf import settings from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate +from M2Crypto import SMIME, BIO, X509 -from django.conf import settings from tags import SoftwareTagList, SoftwareTag import osandarch @@ -196,12 +194,14 @@ def parsePackageMetadata(packageFile): foundInfo = False foundIcon = False digest = hashlib.new('sha256') - #Init magic sequnce checker + #Init magic sequence checker ms = magic.Magic() osset = set() archset = set() pkgfmt = set() + packageHeaders = ['am-application', 'am-package'] + for entry in pkg: fileCount = fileCount + 1 @@ -234,9 +234,12 @@ def parsePackageMetadata(packageFile): if len(docs) != 2: raise Exception('file --PACKAGE-HEADER-- does not consist of 2 YAML documents') - if docs[0]['formatVersion'] != 1 or docs[0]['formatType'] != 'am-package-header': + if not (docs[0]['formatVersion'] in [1, 2] and docs[0]['formatType'] == 'am-package-header'): raise Exception('file --PACKAGE-HEADER-- has an invalid document type') + # Set initial package format version from --PACKAGE-HEADER-- + # it must be consistent with info.yaml file + pkgdata['packageFormat'] = docs[0] pkgdata['header'] = docs[1] elif fileCount == 1: raise Exception('the first file in the package is not --PACKAGE-HEADER--, but %s' % entry.name) @@ -271,10 +274,13 @@ def parsePackageMetadata(packageFile): if len(docs) != 2: raise Exception('file %s does not consist of 2 YAML documents' % entry.name) - if docs[0]['formatVersion'] != 1 or docs[0]['formatType'] != 'am-application': + if docs[0]['formatVersion'] != 1 or not docs[0]['formatType'] in packageHeaders: raise Exception('file %s has an invalid document type' % entry.name) + if (packageHeaders.index(docs[0]['formatType']) + 1) > pkgdata['packageFormat']['formatVersion']: + raise Exception('inconsistent package version between --PACKAGE-HEADER-- and info.yaml files.') pkgdata['info'] = docs[1] + pkgdata['info.type'] = docs[0]['formatType'] foundInfo = True elif entry.name == 'icon.png': @@ -310,8 +316,10 @@ def parsePackageMetadata(packageFile): if len(docs) < 2: raise Exception('file --PACKAGE-FOOTER-- does not consist of at least 2 YAML documents') - if docs[0]['formatVersion'] != 1 or docs[0]['formatType'] != 'am-package-footer': + if not (docs[0]['formatVersion'] in [1, 2]) or docs[0]['formatType'] != 'am-package-footer': raise Exception('file --PACKAGE-FOOTER-- has an invalid document type') + if docs[0]['formatVersion'] != pkgdata['packageFormat']['formatVersion']: + raise Exception('inconsistent package version between --PACKAGE-HEADER-- and --PACKAGE-FOOTER-- files.') pkgdata['footer'] = docs[1] for doc in docs[2:]: @@ -326,7 +334,7 @@ def parsePackageMetadata(packageFile): raise Exception('Multiple binary architectures detected in package') if len(pkgfmt) > 1: raise Exception('Multiple binary formats detected in package') - if (len(osset) == 0) and (len(archset) == 0) and (len(pkgfmt) == 0): + if (not osset) and (not archset) and (not pkgfmt): pkgdata['architecture'] = 'All' else: pkgdata['architecture'] = list(archset)[0] @@ -339,8 +347,24 @@ def parsePackageMetadata(packageFile): def parseAndValidatePackageMetadata(packageFile, certificates = []): pkgdata = parsePackageMetadata(packageFile) - partFields = { 'header': [ 'applicationId', 'diskSpaceUsed' ], - 'info': [ 'id', 'name', 'icon', 'runtime', 'code' ], + if pkgdata['packageFormat']['formatVersion'] == 1: + packageIdKey = 'applicationId' + elif pkgdata['packageFormat']['formatVersion'] == 2: + packageIdKey = 'packageId' + else: + raise Exception('Unknown package formatVersion %s' % pkgdata['packageFormat']['formatVersion']) + + if pkgdata['info.type'] == 'am-package': + infoList = ['id', 'name', 'icon', 'applications'] + elif pkgdata['info.type'] == 'am-application': + infoList = ['id', 'name', 'icon', 'runtime', 'code'] + else: + raise Exception('Unknown info.yaml formatType %s' % pkgdata['info.type']) + + + + partFields = { 'header': [ packageIdKey, 'diskSpaceUsed' ], + 'info': infoList, 'footer': [ 'digest' ], 'icon': [], 'digest': [] } @@ -354,8 +378,8 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): if field not in data: raise Exception('metadata %s is missing in the %s part' % (field, part)) - if pkgdata['header']['applicationId'] != pkgdata['info']['id']: - raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header']['applicationId'], pkgdata['info']['id'])) + if pkgdata['header'][packageIdKey] != pkgdata['info']['id']: + raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header'][packageIdKey], pkgdata['info']['id'])) error = [''] if not isValidDnsName(pkgdata['info']['id'], error): @@ -375,13 +399,13 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): elif len(pkgdata['info']['name']) > 0: name = pkgdata['info']['name'].values()[0] - if len(name) == 0: + if not name: raise Exception('could not deduce a suitable package name from the info part') pkgdata['storeName'] = name if pkgdata['digest'] != pkgdata['footer']['digest']: - raise Exception('digest does not match, is: %s, but should be %s' % (pkgdata['digest'], pkgdata['footer']['digest'])) + raise Exception('digest does not match, is: %s, but should be %s' % (pkgdata['digest'], pkgdata['footer']['digest'])) if 'storeSignature' in pkgdata['footer']: raise Exception('cannot upload a package with an existing storeSignature field') @@ -422,14 +446,18 @@ def addFileToPackage(sourcePackageFile, destinationPackageFile, fileName, fileCo dst.close() src.close() -def addSignatureToPackage(sourcePackageFile, destinationPackageFile, digest, deviceId): +def addSignatureToPackage(sourcePackageFile, destinationPackageFile, digest, deviceId, version=1): signingCertificate = '' with open(settings.APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE) as cert: signingCertificate = cert.read() - digestPlusId = hmac.new(deviceId, digest, hashlib.sha256).digest(); - signature = createSignature(digestPlusId, signingCertificate, settings.APPSTORE_STORE_SIGN_PKCS12_PASSWORD) + digestPlusId = hmac.new(deviceId, digest, hashlib.sha256).digest() + signature = createSignature(digestPlusId, signingCertificate, + settings.APPSTORE_STORE_SIGN_PKCS12_PASSWORD) - yamlContent = yaml.dump_all([{ 'formatVersion': 1, 'formatType': 'am-package-footer'}, { 'storeSignature': base64.encodestring(signature) }], explicit_start=True) + yamlContent = yaml.dump_all([{'formatVersion': version, 'formatType': 'am-package-footer'}, + {'storeSignature': base64.encodestring(signature)}], + explicit_start=True) - addFileToPackage(sourcePackageFile, destinationPackageFile, '--PACKAGE-FOOTER--store-signature', yamlContent) + addFileToPackage(sourcePackageFile, destinationPackageFile, + '--PACKAGE-FOOTER--store-signature', yamlContent) From 528d9f6fd0aee70f3dd8c9c01d1d48df980eb8e2 Mon Sep 17 00:00:00 2001 From: Evgeniy Zabotkin Date: Thu, 5 Mar 2020 11:52:51 +0300 Subject: [PATCH 34/48] Appman-packager options to assign tags Change-Id: Id94559a8ad9e4626e25880bf70ae4e641ea20343 Task-number: AUTOSUITE-1231 Reviewed-by: Egor Nemtsev Reviewed-by: Nikolay Zamotaev --- doc/src/deployment-server.qdoc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 3c5000d..a5a02d0 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -94,6 +94,15 @@ version "0.0.0" is assigned. \endlist + \note In order to create a tagged package you can use appman-packager with \c{-m} parameter, for example: + \code + appman-packager create-package package.pkg package-tmp/apps/com.luxoft.alexa/ -m " + {tags: ['platform:desktop']}" --verbose + \endcode + + This command creates package and adds \c{platform} tag with \c{desktop} value to it. + For reference see the \l{Packager} section of \l{Qt Application Manager} documentation. + \section1 Installation \section2 Set up the Server in a Virtual Environment From 830bb18addd3dfa4a868c25d143d93a112f4dfa2 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 18 Mar 2020 18:05:07 +0300 Subject: [PATCH 35/48] Module version bump Fixes: AUTOSUITE-1543 Change-Id: I702374f5f50a3e3ac7b34bbf9dfee197b808d722 Reviewed-by: Nikolay Zamotaev --- .qmake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.qmake.conf b/.qmake.conf index 9bdff44..307d672 100644 --- a/.qmake.conf +++ b/.qmake.conf @@ -1,3 +1,3 @@ -MODULE_VERSION = 5.12 +MODULE_VERSION = 5.14.1 CONFIG += prepare_docs qt_docs_targets From 7b16ce74b5efd12d307a511135b21eceae764b97 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Thu, 2 Apr 2020 18:05:59 +0300 Subject: [PATCH 36/48] Category icon decolorisation disabled by default Change-Id: I90b2d68b3f7ee21efaba19d5d0d7a1a4d1a4f314 Fixes: AUTOSUITE-1462 Reviewed-by: Evgeniy Zabotkin Reviewed-by: Egor Nemtsev --- appstore/settings.py | 6 +++--- doc/src/deployment-server.qdoc | 10 ++++++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/appstore/settings.py b/appstore/settings.py index 2224284..092ea09 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -159,8 +159,8 @@ MEDIA_URL = '' # Icon size (icons are resized to this size on upload) -ICON_SIZE_X = 36 -ICON_SIZE_Y = 32 +ICON_SIZE_X = 50 +ICON_SIZE_Y = 50 # If the icon should be transformed to monochrome, with alpha channel, when uploaded or not -ICON_DECOLOR = True +ICON_DECOLOR = False diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index a5a02d0..0afd215 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -118,6 +118,16 @@ Make sure to adapt the \c{APPSTORE_*} settings in \c{appstore/settings.py} to your environment, before you run the server. + One setting group to note is category icon resizing and decolorization settings. They are: + \c{ICON_SIZE_X}, \c{ICON_SIZE_Y} and \c{ICON_DECOLOR}. \c{ICON_DECOLOR} is False by default, and it + regulates stripping the category icon of color and alpha channel and the subsequent application + of the same image as an alpha channel instead. + + \c{ICON_SIZE_X} and \c{ICON_SIZE_Y} are the maximum icon sizes, icons are scaled in such a way, + as not to exceed this size, while keeping the aspect ratio of the icon. Default values are 50x50, + so you should be aware that any uploaded icon will be limited to this size, unless the server is + reconfigured. + Since package downloads are done via temporary files, you need to setup a cron-job to remove these temporary files periodically. The cron-job should be triggered every \c{settings.APPSTORE_DOWNLOAD_EXPIRY/2} minutes; it needs to run: From 6d3df9e35af6d3d3eafbf23860c9d7870fd6cde9 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 10 Apr 2020 16:51:38 +0300 Subject: [PATCH 37/48] Module version update Change-Id: I29d015c2ae400a2e8e7a2e78d2d902e744abe807 Reviewed-by: Svetlana Abramenkova --- .qmake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.qmake.conf b/.qmake.conf index 307d672..68f414a 100644 --- a/.qmake.conf +++ b/.qmake.conf @@ -1,3 +1,3 @@ -MODULE_VERSION = 5.14.1 +MODULE_VERSION = 5.14.2 CONFIG += prepare_docs qt_docs_targets From e599d4bae12469f5252be88944d4982ec70d6302 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 17 Jun 2020 13:02:32 +0300 Subject: [PATCH 38/48] Bump version to 5.15.0 Change-Id: I9f083641f1d2ba1707554b9127e8ed49898bb394 Fixes: AUTOSUITE-1590 Reviewed-by: Nikolay Zamotaev --- .qmake.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.qmake.conf b/.qmake.conf index 68f414a..86b26fa 100644 --- a/.qmake.conf +++ b/.qmake.conf @@ -1,3 +1,3 @@ -MODULE_VERSION = 5.14.2 +MODULE_VERSION = 5.15.0 CONFIG += prepare_docs qt_docs_targets From e3be769efcea0520a161700bb4ba75f8bcc2274c Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 22 Jul 2020 19:54:16 +0300 Subject: [PATCH 39/48] Fix for vendor duplication when changing the user Task-number: AUTOSUITE-1463 Change-Id: Id6c21ad88618513766c5b5de8e75c40c258f8828 Reviewed-by: Egor Nemtsev --- store/migrations/0001_initial.py | 7 ++++--- store/models.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index d0e2c04..f23c6bf 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -31,7 +31,7 @@ ## ############################################################################# -# Generated by Django 1.11.27 on 2020-02-07 16:50 +# Generated by Django 1.11.27 on 2020-07-22 16:39 from __future__ import unicode_literals from django.conf import settings @@ -45,7 +45,7 @@ class Migration(migrations.Migration): initial = True dependencies = [ - ('auth', '0008_alter_user_username_max_length'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), ] operations = [ @@ -82,9 +82,10 @@ class Migration(migrations.Migration): migrations.CreateModel( name='Vendor', fields=[ - ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)), + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('name', models.CharField(max_length=200)), ('certificate', models.TextField(max_length=8000)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], ), migrations.AddField( diff --git a/store/models.py b/store/models.py index 93136af..1cf8916 100644 --- a/store/models.py +++ b/store/models.py @@ -75,7 +75,7 @@ def save(self, *args, **kwargs): super(Category, self).save(*args, **kwargs) class Vendor(models.Model): - user = models.ForeignKey(User, primary_key = True) + user = models.ForeignKey(User, primary_key = False) name = models.CharField(max_length = 200) certificate = models.TextField(max_length = 8000) From 4f9f1fc171601d151ee0208efbfd0ffa9e73b501 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Mon, 27 Jul 2020 17:10:18 +0300 Subject: [PATCH 40/48] Architecture format parsing fix Fix parsing architecture to take into account extra components that can be sent by QSysInfo::buildAbi(). Updated documentation to reflect it. The file: doc/src/deployment-server-reference.qdoc was not changed, because it reflects server output format, not input parameter. Change-Id: I1cdcac0d47f0e5f75db061b871c749820614fd3c Fixes: AUTOSUITE-1426 Reviewed-by: Egor Nemtsev --- doc/src/deployment-server.qdoc | 3 ++- store/osandarch.py | 14 ++++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 0afd215..39d11dd 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -68,7 +68,8 @@ these groups are unique. \li Architecture is specified as a group of: CPU architecture, endianness, bitness, and OS. If a package does not contain architecture specific parts, the architecture is specified as \e{All}. - \li CPU architecture is based on the return value from QsysInfo::buildCpuArchitecture(). + \li CPU architecture is based on the return value from QsysInfo::buildAbi() and + QSysInfo::kernelType(), joined with '-'. \li The installation target is automatically determined by parsing binary files. For example, detecting an ELF binary means that it's a Linux package; the CPU architecture, such as armv8, further defines the installation target. diff --git a/store/osandarch.py b/store/osandarch.py index 0dde800..7b833d3 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -134,7 +134,8 @@ def getOsArch(str): def normalizeArch(inputArch): """ This function brings requested architecture to common form (currently just parses the bits part and turns it into 32/64) - Input string format is: arch-endianness-word_size-kernelType + Input string format is: arch-endianness-word_size-optional_ABI-kernelType + Output string format is: arch-endianness-word_size-kernelType """ parts = inputArch.split('-') @@ -143,15 +144,16 @@ def normalizeArch(inputArch): #Drop anything non-numeric from word_size field parts[2]=re.sub(r"\D", "", parts[2]) #Transform kernelType into binary format - temp = parts[3] - if "win" in temp: - parts[3]="pe32" + temp = parts[-1] #the last element is kernel type + if "darwin" in temp: + parts[3]="mach_o" elif "linux" in temp: parts[3]="elf" elif "freebsd" in temp: #How do we treat QNX? parts[3]="elf" - elif "darwin" in temp: - parts[3]="mach_o" + elif "win" in temp: + parts[3]="pe32" + parts=parts[0:4] # now format drops optional part #Rejoin new architecture arch = '-'.join(parts) return arch From 23450bceb6b9d032b7444ab25947d405dcb441d8 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Thu, 20 Feb 2020 17:13:17 +0300 Subject: [PATCH 41/48] Implement reverse tag matching Fixes: AUTOSUITE-1447 Change-Id: I7f8c3c83d49939962376c98a650b4ecb24f8e980 Reviewed-by: Nikolay Zamotaev --- appstore/settings.py | 4 +- appstore/urls.py | 7 +- appstore/wsgi.py | 3 +- doc/src/deployment-server-reference.qdoc | 28 ++- store/admin.py | 59 +++-- store/api.py | 204 ++++++++++++------ .../commands/store-upload-package.py | 31 +-- store/migrations/0001_initial.py | 26 ++- store/models.py | 111 +++++++--- store/osandarch.py | 54 ++--- store/tags.py | 131 ++++------- store/utilities.py | 35 +-- 12 files changed, 421 insertions(+), 272 deletions(-) diff --git a/appstore/settings.py b/appstore/settings.py index 092ea09..b03e943 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -39,6 +39,7 @@ For the full list of settings and their values, see https://docs.djangoproject.com/en/1.7/ref/settings/ """ +import os APPSTORE_MAINTENANCE = False APPSTORE_PLATFORM_ID = 'NEPTUNE3' @@ -50,10 +51,9 @@ APPSTORE_NO_SECURITY = True # ignore developer signatures and do not generate store signatures APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE = 'certificates/store.p12' APPSTORE_STORE_SIGN_PKCS12_PASSWORD = 'password' -APPSTORE_DEV_VERIFY_CA_CERTIFICATES = [ 'certificates/ca.crt', 'certificates/devca.crt' ] +APPSTORE_DEV_VERIFY_CA_CERTIFICATES = ['certificates/ca.crt', 'certificates/devca.crt'] # Build paths inside the project like this: os.path.join(BASE_DIR, ...) -import os BASE_DIR = os.path.dirname(os.path.dirname(__file__)) diff --git a/appstore/urls.py b/appstore/urls.py index 7c99a6a..1eb3d2c 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -33,7 +33,7 @@ from django.conf.urls import include, url from django.contrib import admin from store import api as store_api -import settings +from appstore.settings import URL_PREFIX base_urlpatterns = [ url(r'^admin/', include(admin.site.urls)), @@ -42,6 +42,7 @@ url(r'^login$', store_api.login), url(r'^logout$', store_api.logout), url(r'^app/list$', store_api.appList), + url(r'^app/icons/(.*)$', store_api.appIconNew), url(r'^app/icon', store_api.appIcon), url(r'^app/description', store_api.appDescription), url(r'^app/purchase', store_api.appPurchase), @@ -53,8 +54,8 @@ prefix = '^' -if settings.URL_PREFIX !='': - prefix = prefix + settings.URL_PREFIX + '/' +if URL_PREFIX != '': + prefix = prefix + URL_PREFIX + '/' urlpatterns = [ url(prefix, include(base_urlpatterns)), diff --git a/appstore/wsgi.py b/appstore/wsgi.py index 3f007d4..5ff13a6 100644 --- a/appstore/wsgi.py +++ b/appstore/wsgi.py @@ -40,7 +40,8 @@ """ import os +from django.core.wsgi import get_wsgi_application + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "appstore.settings") -from django.core.wsgi import get_wsgi_application application = get_wsgi_application() diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index 1ade88f..a8903de 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -55,15 +55,10 @@ \li The Deployment Server's HTTP API version that you are using to communicate with the server. For more information, see \c{settings.APPSTORE_VERSION}. \row - \li require_tag + \li require_tag (can also be passed as \c{tag} ) \li An optional parameter used to filter packages by tags. Receives a comma-separated - list of tags; these tags must be alphanumeric. Only applications that contain any - of the specified tags should be listed. - \row - \li conflicts_tag - \li An optional parameter used to filter packages by tags. Receives a comma-separated - list of tags; these tags must be alphanumeric. Applications that contain any of the - speicifed tags should be excluded. + list of tags; these tags must be alphanumeric. Only applications that match any of the + specified tags (and not matching those in their conflicts list should be listed). \row \li architecture \li An optional parameter used to filter packages by architecture. Receives the CPU @@ -190,7 +185,10 @@ \li A category name for the app. \row \li tags - \li JSON array of app tags + \li JSON array of tags required for the app + \row + \li conflict_tags + \li JSON array of tags which will conflict with the app \row \li version \li The app's version, returned as a string. If the there is no version number, the @@ -216,6 +214,14 @@ \row \li category_id \li Numeric category ID that matches the app's category. + \row + \li purchaseId + \li Purchase identifier, freeform identifier of specific application on the server. + Used as an alternative to ID in \c{app/purchase} API call. This is a freeform string, + currently a UUID, but no assumptions should be made of the format. + \row + \li iconUrl + \li URL for the icon of this application \endtable \section2 app/icon @@ -270,6 +276,10 @@ \row \li id \li The app ID. + \row + \li purchaseId + \li Alternative app ID, to select specific app with tags and all (see \c{app/list} API description). + If both ID and purchaseId are specified, ID takes precedence. \endtable Returns a JSON object: \table diff --git a/store/admin.py b/store/admin.py index 2252af1..c55f6d4 100644 --- a/store/admin.py +++ b/store/admin.py @@ -30,7 +30,6 @@ ## ############################################################################# -import os import StringIO from PIL import Image, ImageChops @@ -42,7 +41,7 @@ from ordered_model.admin import OrderedModelAdmin from store.models import * -from utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList +from store.utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList class CategoryAdminForm(forms.ModelForm): class Meta: @@ -60,14 +59,14 @@ def clean(self): # make image monochrome + alpha channel, this is done to compensate for # how icons are treated in neptune3-ui im = Image.open(cleaned_data['icon']) - grey, alpha = im.convert('LA').split() + grey, _ = im.convert('LA').split() grey = ImageChops.invert(grey) im.putalpha(grey) im = im.convert('LA') else: # No conversion, icons are uploaded as-is, only scaling is used. im = Image.open(cleaned_data['icon']) - size = (settings.ICON_SIZE_X,settings.ICON_SIZE_Y,) + size = (settings.ICON_SIZE_X, settings.ICON_SIZE_Y) im.thumbnail(size, Image.ANTIALIAS) imagefile = StringIO.StringIO() im.save(imagefile, format='png') @@ -100,41 +99,48 @@ def icon_image(self, obj): class AppAdminForm(forms.ModelForm): class Meta: - exclude = ["appid", "name", "tags", "architecture", 'version', 'pkgformat'] + exclude = ["appid", "name", "tags", "tags_hash", "architecture", 'version', 'pkgformat'] appId = "" name = "" def clean(self): cleaned_data = super(AppAdminForm, self).clean() - file = cleaned_data.get('file') + package_file = cleaned_data.get('file') # validate package try: - pkgdata = parseAndValidatePackageMetadata(file) + pkgdata = parseAndValidatePackageMetadata(package_file) except Exception as error: raise forms.ValidationError(_('Validation error: %s' % str(error))) self.appId = pkgdata['info']['id'] self.name = pkgdata['storeName'] self.architecture = pkgdata['architecture'] - self.tags = makeTagList(pkgdata) + require_tags, conflict_tags, self.tags_hash = makeTagList(pkgdata) # check if this really is an update if hasattr(self, 'instance') and self.instance.appid: if self.appId != self.instance.appid: - raise forms.ValidationError(_('Validation error: an update cannot change the application id, tried to change from %s to %s' % (self.instance.appid, self.appId))) + raise forms.ValidationError(_('Validation error: an update cannot change the ' + 'application id, tried to change from %s to %s' % + (self.instance.appid, self.appId))) elif self.architecture != self.instance.architecture: - raise forms.ValidationError(_('Validation error: an update cannot change the application architecture from %s to %s' % (self.instance.architecture, self.architecture))) + raise forms.ValidationError(_('Validation error: an update cannot change the ' + 'application architecture from %s to %s' % + (self.instance.architecture, self.architecture))) else: try: - if App.objects.get(appid__exact = self.appId, architecture__exact = self.architecture, tags__exact = self.tags): - raise forms.ValidationError(_('Validation error: another application with id %s , tags %s and architecture %s already exists' % (str(self.appId), str(self.tags), str(self.architecture)))) + if App.objects.get(appid__exact=self.appId, architecture__exact=self.architecture, tags_hash__exact=self.tags_hash): + raise forms.ValidationError(_('Validation error: another application with id' + ' %s , tags %s and architecture %s already ' + 'exists' % (str(self.appId), str(self.tags_hash), + str(self.architecture)))) except App.DoesNotExist: pass # write icon into file to serve statically - success, error = writeTempIcon(self.appId, self.architecture, self.tags, pkgdata['icon']) + success, error = writeTempIcon(self.appId, self.architecture, self.tags_hash, pkgdata['icon']) if not success: raise forms.ValidationError(_(error)) @@ -148,19 +154,42 @@ def save(self, commit=False): m.file.seek(0) pkgdata = parseAndValidatePackageMetadata(m.file) - m.tags = makeTagList(pkgdata) + tags, conflict_tags, m.tags_hash = makeTagList(pkgdata) + + # FIXME - add tags? + taglist = populateTagList(tags.list(),conflict_tags.list()) + + # FIXME: clean tags beforehand m.pkgformat = pkgdata['packageFormat']['formatVersion'] + m.save() + + for i in taglist: + # attach tags to app + m.tags.add(i) + return m class AppAdmin(admin.ModelAdmin): form = AppAdminForm - list_display = ('name', 'appid', 'architecture', 'version', 'pkgformat', 'tags') + list_display = ('name', 'appid', 'architecture', 'version', 'pkgformat', 'tags_hash') def save_model(self, request, obj, form, change): obj.save() +class TagAdmin(admin.ModelAdmin): + list_display = ('negative', 'name', 'version') + + def has_add_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + def save_model(self, request, obj, form, change): + obj.save() admin.site.register(Category, CategoryAdmin) admin.site.register(Vendor) +admin.site.register(Tag, TagAdmin) admin.site.register(App, AppAdmin) diff --git a/store/api.py b/store/api.py index fe8e25d..75e730f 100644 --- a/store/api.py +++ b/store/api.py @@ -39,14 +39,16 @@ from django.http import HttpResponse, HttpResponseForbidden, Http404, JsonResponse from django.contrib import auth from django.views.decorators.csrf import csrf_exempt -from authdecorators import logged_in_or_basicauth, is_staff_member +from django.core.exceptions import ValidationError +from store.authdecorators import logged_in_or_basicauth, is_staff_member -from models import App, Category, Vendor, savePackageFile -from utilities import parsePackageMetadata, parseAndValidatePackageMetadata, addSignatureToPackage -from utilities import packagePath, iconPath, downloadPath -from utilities import getRequestDictionary -from osandarch import normalizeArch -from tags import SoftwareTagList +from store.models import App, Category, Vendor, savePackageFile +from store.utilities import parsePackageMetadata, parseAndValidatePackageMetadata, \ + addSignatureToPackage +from store.utilities import packagePath, iconPath, downloadPath +from store.utilities import getRequestDictionary +from store.osandarch import normalizeArch +from store.tags import SoftwareTagList def hello(request): @@ -66,16 +68,21 @@ def hello(request): elif not ((version) > 0 and (version <= settings.APPSTORE_PLATFORM_VERSION)): status = 'incompatible-version' - for j in ("require_tag", "conflicts_tag",): - if j in dictionary: #Tags are coma-separated, - versionmap = SoftwareTagList() - if not versionmap.parse(getRequestDictionary(request)[j]): - status = 'malformed-tag' - break - request.session[j] = str(versionmap) + # Tag parsing + temp_tags = [] + for j in ("require_tag", "tag"): + if j in dictionary: + temp_tags.append(dictionary[j]) + versionmap = SoftwareTagList() + #Tags are coma-separated, - so join them + if temp_tags: + if not versionmap.parse(','.join(temp_tags)): + status = 'malformed-tag' + request.session["tag"] = str(versionmap) + del temp_tags if 'architecture' in dictionary: - arch = normalizeArch(getRequestDictionary(request)['architecture']) + arch = normalizeArch(dictionary['architecture']) if arch == "": status = 'incompatible-architecture' request.session['architecture'] = arch @@ -95,7 +102,7 @@ def login(request): except KeyError: raise Exception('missing-credentials') - user = auth.authenticate(username = username, password = password) + user = auth.authenticate(username=username, password=password) if user is None: raise Exception('authentication-failed') @@ -147,9 +154,9 @@ def upload(request): myfile = request.FILES['package'] category = Category.objects.all().filter(name__exact=category_name) vendor = Vendor.objects.all().filter(name__exact=vendor_name) - if len(category) == 0: + if not category: raise Exception('Non-existing category') - if len(vendor) == 0: + if not vendor: raise Exception('Non-existing vendor') try: @@ -159,7 +166,11 @@ def upload(request): myfile.seek(0) try: - savePackageFile(pkgdata, myfile, category[0], vendor[0], description, shortdescription) + package_metadata = {'category':category[0], + 'vendor':vendor[0], + 'description':description, + 'short_description':shortdescription} + savePackageFile(pkgdata, myfile, package_metadata) except Exception as error: raise Exception(error) else: @@ -174,29 +185,15 @@ def appList(request): apps = App.objects.all() dictionary = getRequestDictionary(request) if 'filter' in dictionary: - apps = apps.filter(name__contains = dictionary['filter']) + apps = apps.filter(name__contains=dictionary['filter']) if 'category_id' in dictionary: catId = dictionary['category_id'] if catId != -1: # All metacategory - apps = apps.filter(category__exact = catId) - - #Tag filtering - #"require_tag", "conflicts_tag" - # Tags are combined by logical AND (for require) and logical OR for conflicts - if 'require_tag' in request.session: - require_tags = SoftwareTagList() - require_tags.parse(request.session['require_tag']) - for i in require_tags.make_regex(): - apps = apps.filter(Q(tags__regex = i)) - if 'conflicts_tag' in request.session: - conflict_tags = SoftwareTagList() - conflict_tags.parse(request.session['conflicts_tag']) - for i in conflict_tags.make_regex(): - apps = apps.filter(~Q(tags__regex=i)) + apps = apps.filter(category__exact=catId) # Here goes the logic of listing packages when multiple architectures are available - # in /hello request, the target architecture is stored in the session. By definition target machine can support - # both "All" package architecture and it's native one. + # in /hello request, the target architecture is stored in the session. By definition + # target machine can support both "All" package architecture and it's native one. # So - here goes filtering by list of architectures archlist = ['All', ] if 'architecture' in request.session: @@ -206,52 +203,93 @@ def appList(request): if 'pkgversions' in request.session: versionlist = request.session['pkgversions'] - apps = apps.filter(architecture__in = archlist) - apps = apps.filter(pkgformat__in = versionlist) - - # After filtering, there are potential duplicates in list. And we should prefer native applications to pure qml ones - # due to speedups offered. - # So - first applications are grouped by appid and numbers of same appids counted. In case where is more than one appid - - # there are two variants of application: for 'All' architecture, and for the architecture supported by the target machine. - # So, as native apps should be preferred - duplicates = ( - apps.values('appid').order_by().annotate(count_id=Count('id')).filter(count_id__gt=1) - ) - # Here we go over duplicates list and filter out 'All' architecture apps. - for duplicate in duplicates: - apps = apps.filter(~Q(appid__exact = duplicate['appid'], architecture__exact = 'All')) # if there is native - 'All' architecture apps are excluded - - appList = list(apps.values('appid', 'name', 'vendor__name', 'briefDescription', 'category', 'tags', 'architecture', 'version').order_by('appid','architecture')) + apps = apps.filter(architecture__in=archlist) + apps = apps.filter(pkgformat__in=versionlist) - for app in appList: + #Tag filtering + #There is no search by version distance yet - this must be fixed + if 'tag' in request.session: + tags = SoftwareTagList() + tags.parse(request.session['tag']) + apps_ids = [app.id for app in apps if app.is_tagmatching(tags.list())] + apps = App.objects.filter(id__in=apps_ids) + + # After filtering, there are potential duplicates in list. And we should prefer native + # applications to pure qml ones due to speedups offered. + # At this point - filtering duplicates is disabled, because it will be implemented with + # 'distance' between requested version and package version. Architecture will be also included + # in this metric (as native apps should be preferred) + + selectedAppList = apps.values('id', 'appid', 'name', 'vendor__name', 'briefDescription', + 'category', 'architecture', + 'version', 'pkgformat', 'tags_hash').order_by('appid', 'architecture', 'tags_hash') + + appList = [] + for app in selectedAppList: + app['purchaseId'] = app['id'] app['id'] = app['appid'] app['category_id'] = app['category'] - app['category'] = Category.objects.all().filter(id__exact = app['category_id'])[0].name + app['category'] = Category.objects.all().filter(id__exact=app['category_id'])[0].name app['vendor'] = app['vendor__name'] - if app['tags'] != "": - app['tags'] = app['tags'].split(',') + app['tags'] = [] + tags = list(App.objects.filter(appid=app['appid'], architecture=app['architecture'], tags_hash=app['tags_hash'], tags__negative=False).values('tags__name', 'tags__version')) + for i in tags: + app['tags'].append(i['tags__name'] if not i['tags__version'] else ':'.join((i['tags__name'],i['tags__version']))) + app['conflict_tags'] = [] + conflict_tags = list(App.objects.filter(appid=app['appid'], architecture=app['architecture'], tags_hash=app['tags_hash'], tags__negative=True).values('tags__name', 'tags__version')) + for i in conflict_tags: + app['conflict_tags'].append(i['tags__name'] if not i['tags__version'] else ':'.join((i['tags__name'],i['tags__version']))) + + toFile = '_'.join([app['appid'], app['architecture'], app['tags_hash']]) + if settings.URL_PREFIX != '': + iconUri = '/' + settings.URL_PREFIX + '/app/icons/' + toFile else: - app['tags'] = [] + iconUri = '/app/icons/' + toFile + app['iconUrl'] = request.build_absolute_uri(iconUri) del app['vendor__name'] del app['appid'] + del app['tags_hash'] + appList.append(app) # this is not valid JSON, since we are returning a list! - return JsonResponse(appList, safe = False) + return JsonResponse(appList, safe=False) def appDescription(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) + versionlist = [1] + if 'pkgversions' in request.session: + versionlist = request.session['pkgversions'] appId = getRequestDictionary(request)['id'] try: - app = App.objects.get(appid__exact = appId, architecture__in = archlist).order_by('architecture') + app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') + app = app.filter(pkgformat__in=versionlist) + #Tag filtering + #There is no search by version distance yet - this must be fixed + if 'tag' in request.session: + tags = SoftwareTagList() + tags.parse(request.session['tag']) + app_ids = [x.id for x in app if x.is_tagmatching(tags.list())] + app = App.objects.filter(id__in=app_ids) app = app.last() return HttpResponse(app.description) except: raise Http404('no such application: %s' % appId) +def appIconNew(request, path): + path=path.replace('/', '_').replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png' + try: + response = HttpResponse(content_type='image/png') + with open(iconPath() + path, 'rb') as pkg: + response.write(pkg.read()) + response['Content-Length'] = pkg.tell() + return response + except: + raise Http404 + def appIcon(request): archlist = ['All', ] dictionary = getRequestDictionary(request) @@ -259,12 +297,23 @@ def appIcon(request): archlist.append(normalizeArch(dictionary['architecture'])) elif 'architecture' in request.session: archlist.append(request.session['architecture']) + versionlist = [1] + if 'pkgversions' in request.session: + versionlist = request.session['pkgversions'] appId = dictionary['id'] try: - app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture') + app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') + app = app.filter(pkgformat__in=versionlist) + #Tag filtering + #There is no search by version distance yet - this must be fixed + if 'tag' in request.session: + tags = SoftwareTagList() + tags.parse(request.session['tag']) + app_ids = [x.id for x in app if x.is_tagmatching(tags.list())] + app = App.objects.filter(id__in=app_ids) app = app.last() - with open(iconPath(app.appid,app.architecture,app.tags), 'rb') as iconPng: - response = HttpResponse(content_type = 'image/png') + with open(iconPath(app.appid, app.architecture, app.tags_hash), 'rb') as iconPng: + response = HttpResponse(content_type='image/png') response.write(iconPng.read()) return response except: @@ -277,6 +326,10 @@ def appPurchase(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) + versionlist = [1] + if 'pkgversions' in request.session: + versionlist = request.session['pkgversions'] + try: deviceId = str(getRequestDictionary(request).get("device_id", "")) if settings.APPSTORE_BIND_TO_DEVICE_ID: @@ -285,13 +338,28 @@ def appPurchase(request): else: deviceId = '' - app = App.objects.filter(appid__exact = getRequestDictionary(request)['id'], architecture__in=archlist).order_by('architecture') + if 'id' in getRequestDictionary(request): + app = App.objects.filter(appid__exact = getRequestDictionary(request)['id'], architecture__in=archlist).order_by('architecture','tags_hash') + elif 'purchaseId' in getRequestDictionary(request): + app = App.objects.filter(id__exact = getRequestDictionary(request)['purchaseId'], architecture__in=archlist).order_by('architecture','tags_hash') + else: + raise ValidationError('id or purchaseId parameter required') + app = app.filter(pkgformat__in=versionlist) + #Tag filtering + #There is no search by version distance yet - this must be fixed + if 'tag' in request.session: + tags = SoftwareTagList() + tags.parse(request.session['tag']) + app_ids = [x.id for x in app if x.is_tagmatching(tags.list())] + app = App.objects.filter(id__in=app_ids) + app = app.last() - fromFilePath = packagePath(app.appid, app.architecture, app.tags) + fromFilePath = packagePath(app.appid, app.architecture, app.tags_hash) # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) - toFile = str(app.appid) + '_' + str(request.user.id) + '_' + str(app.architecture) + '_' + str(app.tags) + '_'+ str(deviceId) + toFile = '_'.join((str(app.appid), str(request.user.id), str(app.architecture), + str(app.tags_hash), str(deviceId))) if not settings.DEBUG: toFile = hashlib.sha256(toFile).hexdigest() toFile += '.appkg' @@ -309,7 +377,7 @@ def appPurchase(request): shutil.copyfile(fromFilePath, toPath + toFile) except Exception as error: if type(error) == IOError: - raise IOError(error.args[0],error.args[1], os.path.basename(fromFilePath)) + raise IOError(error.args[0], error.args[1], os.path.basename(fromFilePath)) else: raise error @@ -330,7 +398,7 @@ def appPurchase(request): def appDownload(request, path): try: - response = HttpResponse(content_type = 'application/octetstream') + response = HttpResponse(content_type='application/octetstream') with open(downloadPath() + path, 'rb') as pkg: response.write(pkg.read()) response['Content-Length'] = pkg.tell() diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 77286bc..7af866b 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -30,18 +30,22 @@ ## ############################################################################# -import os - +from optparse import make_option from django.core.management.base import BaseCommand, CommandError from django.core.files.base import ContentFile -from store.models import App, Category, Vendor, savePackageFile +from store.models import Category, Vendor, savePackageFile from store.utilities import parseAndValidatePackageMetadata -from optparse import make_option + class Command(BaseCommand): help = 'Uploads a package to the deployment server. This can be used for batch uploading.' - option_list = BaseCommand.option_list + ( + usage_string = 'Usage: manage.py store-upload-package --vendor --category ' \ + ' [--description ] ' + + # FIXME: this doesn't work: + # see https://docs.djangoproject.com/en/1.8/howto/custom-management-commands/#django.core.management.BaseCommand.add_arguments + self.option_list = BaseCommand.option_list + ( make_option('--vendor', action='store', type="string", @@ -62,16 +66,14 @@ class Command(BaseCommand): def handle(self, *args, **options): if len(args) != 1: - raise CommandError( - 'Usage: manage.py store-upload-package --vendor --category [--description ] ') + raise CommandError(self.usage_string) if (not options['vendor']) or (not options['category']): - raise CommandError( - 'Usage: manage.py store-upload-package --vendor --category [--description ] ') + raise CommandError(self.usage_string) category = Category.objects.all().filter(name__exact=options['category']) vendor = Vendor.objects.all().filter(name__exact=options['vendor']) - if len(category) == 0: + if not category: raise CommandError('Non-existing category specified') - if len(vendor) == 0: + if not vendor: raise CommandError('Non-existing vendor specified') try: @@ -86,7 +88,10 @@ def handle(self, *args, **options): packagefile.seek(0) description = options['description'] try: - savePackageFile(pkgdata, ContentFile(packagefile.read()), category[0], vendor[0], description, description) + package_metadata = {'category': category[0], + 'vendor': vendor[0], + 'description': description, + 'short_description': description} + savePackageFile(pkgdata, ContentFile(packagefile.read()), package_metadata) except Exception as error: raise CommandError(error) - diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index f23c6bf..80d154c 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +# Generated by Django 1.11.27 on 2020-08-14 17:24 ############################################################################# ## ## Copyright (C) 2020 Luxoft Sweden AB @@ -38,6 +39,7 @@ from django.db import migrations, models import django.db.models.deletion import store.models +import uuid class Migration(migrations.Migration): @@ -52,7 +54,7 @@ class Migration(migrations.Migration): migrations.CreateModel( name='App', fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('id', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), ('appid', models.CharField(max_length=200)), ('name', models.CharField(max_length=200)), ('file', models.FileField(storage=store.models.OverwriteStorage(), upload_to=store.models.content_file_name)), @@ -60,7 +62,7 @@ class Migration(migrations.Migration): ('description', models.TextField()), ('dateAdded', models.DateField(auto_now_add=True)), ('dateModified', models.DateField(auto_now=True)), - ('tags', models.TextField(blank=True)), + ('tags_hash', models.CharField(default=b'', max_length=4096)), ('architecture', models.CharField(default=b'All', max_length=20)), ('version', models.CharField(default=b'0.0.0', max_length=20)), ('pkgformat', models.IntegerField()), @@ -79,6 +81,15 @@ class Migration(migrations.Migration): 'abstract': False, }, ), + migrations.CreateModel( + name='Tag', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('negative', models.BooleanField(default=False)), + ('name', models.CharField(max_length=200)), + ('version', models.CharField(blank=True, max_length=200)), + ], + ), migrations.CreateModel( name='Vendor', fields=[ @@ -88,11 +99,20 @@ class Migration(migrations.Migration): ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), ], ), + migrations.AlterUniqueTogether( + name='tag', + unique_together=set([('negative', 'name', 'version')]), + ), migrations.AddField( model_name='app', name='category', field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='store.Category'), ), + migrations.AddField( + model_name='app', + name='tags', + field=models.ManyToManyField(to='store.Tag'), + ), migrations.AddField( model_name='app', name='vendor', @@ -100,6 +120,6 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='app', - unique_together=set([('appid', 'architecture', 'tags')]), + unique_together=set([('appid', 'architecture', 'tags_hash')]), ), ] diff --git a/store/models.py b/store/models.py index 1cf8916..7e8751a 100644 --- a/store/models.py +++ b/store/models.py @@ -31,14 +31,16 @@ ############################################################################# import os +import uuid from django.db import models -from ordered_model.models import OrderedModel from django.conf import settings from django.contrib.auth.models import User from django.core.files.storage import FileSystemStorage +from ordered_model.models import OrderedModel -from utilities import packagePath, writeTempIcon, makeTagList +from store.utilities import packagePath, writeTempIcon, makeTagList +from store.tags import SoftwareTag def category_file_name(instance, filename): # filename parameter is unused. See django documentation for details: @@ -66,8 +68,8 @@ def save(self, *args, **kwargs): # This is a django hack. When category icon is saved and then later accessed, # category_id is used as a unique icon identifier. When category is first created, # but not saved yet, category_id is None. So this hack first saves category without icon - # and then saves the icon separately. This is done to prevent creation of category_None.png - # file, when the icon is saved. + # and then saves the icon separately. This is done to prevent creation of + # category_None.png file, when the icon is saved. saved_icon = self.icon self.icon = None super(Category, self).save(*args, **kwargs) @@ -82,76 +84,133 @@ class Vendor(models.Model): def __unicode__(self): return self.name +class Tag(models.Model): + negative = models.BooleanField(default=False) + name = models.CharField(max_length=200) + version = models.CharField(max_length=200, blank=True) + + class Meta: + unique_together = (('negative', 'name', 'version')) + + def __unicode__(self): + negative = '-' if self.negative else '' + if self.version: + return negative + self.name + ":" + self.version + return negative + self.name def content_file_name(instance, filename): - return packagePath(instance.appid, instance.architecture, instance.tags) + return packagePath(instance.appid, instance.architecture, instance.tags_hash) class App(models.Model): - appid = models.CharField(max_length = 200) - name = models.CharField(max_length = 200) - file = models.FileField(upload_to = content_file_name, storage = OverwriteStorage()) + id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + appid = models.CharField(max_length=200) + name = models.CharField(max_length=200) + file = models.FileField(upload_to=content_file_name, storage=OverwriteStorage()) vendor = models.ForeignKey(Vendor) category = models.ForeignKey(Category) briefDescription = models.TextField() description = models.TextField() - dateAdded = models.DateField(auto_now_add = True) - dateModified = models.DateField(auto_now = True) - tags = models.TextField(blank=True) + dateAdded = models.DateField(auto_now_add=True) + dateModified = models.DateField(auto_now=True) + tags = models.ManyToManyField(Tag) + # Hash sum for sorted tag list, this is used to be able to distinguish different tag sets + tags_hash = models.CharField(max_length=4096, default='') architecture = models.CharField(max_length=20, default='All') version = models.CharField(max_length=20, default='0.0.0') pkgformat = models.IntegerField() class Meta: """Makes the group of id and arch - a unique identifier""" - unique_together = (('appid', 'architecture', 'tags'),) + unique_together = (('appid', 'architecture', 'tags_hash'),) def __unicode__(self): - return self.name + " [" + " ".join([self.appid, self.version, self.architecture, self.tags]) + "]" + return self.name + " [" + " ".join([self.appid, self.version, self.architecture, + self.tags_hash]) + "]" + + def is_tagmatching(self, tagstring): + temp = self.tags.all() + if not temp: #App with tags does not require anything + return True + for i in temp: + if not any(j.match(SoftwareTag(str(i))) for j in tagstring): + return False + return True def save(self, *args, **kwargs): try: - this = App.objects.get(appid=self.appid,architecture=self.architecture,tags=self.tags) #FIXME: This should be 'tags match, not exact same tags' + this = App.objects.get(appid=self.appid, architecture=self.architecture, tags_hash=self.tags_hash) if this.file != self.file: this.file.delete(save=False) except: pass super(App, self).save(*args, **kwargs) - -def savePackageFile(pkgdata, pkgfile, category, vendor, description, shortdescription): +def populateTagList(tags, conflict_tags): + taglist = [] + for i in tags: + # Add tag to database + version = "" if i.version is None else i.version + tag, created = Tag.objects.get_or_create(name=i.tag, version=version, negative=False) + if created: + tag.save() + taglist.append(tag) + for i in conflict_tags: + # Add tag to database + version = "" if i.version is None else i.version + tag, created = Tag.objects.get_or_create(name=i.tag, version=version, negative=True) + if created: + tag.save() + return taglist + + +def savePackageFile(pkgdata, pkgfile, package_metadata): appId = pkgdata['info']['id'] name = pkgdata['storeName'] architecture = pkgdata['architecture'] pkgformat = pkgdata['packageFormat']['formatVersion'] - tags = makeTagList(pkgdata) - success, error = writeTempIcon(appId, architecture, tags, pkgdata['icon']) + tags, conflict_tags, tags_hash = makeTagList(pkgdata) + category = package_metadata['category'] + vendor = package_metadata['vendor'] + description = package_metadata['description'] + shortdescription = package_metadata['short_description'] + + success, error = writeTempIcon(appId, architecture, tags_hash, pkgdata['icon']) if not success: raise Exception(error) exists = False app = None try: - app = App.objects.get(appid__exact=appId, architecture__exact=architecture, tags__exact=tags) + app = App.objects.get(appid__exact=appId, architecture__exact=architecture, + tags_hash__exact=tags_hash) exists = True except App.DoesNotExist: pass + taglist = populateTagList(tags.list(), conflict_tags.list()) + if exists: app.appid = appId app.category = category app.vendor = vendor app.name = name - app.tags = tags + app.tags_hash = tags_hash app.description = description app.briefDescription = shortdescription app.architecture = architecture app.pkgformat = pkgformat - app.file.save(packagePath(appId, architecture, tags), pkgfile) + app.file.save(packagePath(appId, architecture, tags_hash), pkgfile) app.save() else: - app, _ = App.objects.get_or_create(name=name, tags=tags, vendor=vendor, - category=category, appid=appId, - briefDescription=shortdescription, description=description, - pkgformat = pkgformat, architecture=architecture) - app.file.save(packagePath(appId, architecture, tags), pkgfile) + app, _ = App.objects.get_or_create(name=name, tags_hash=tags_hash, + vendor=vendor, category=category, appid=appId, + briefDescription=shortdescription, + description=description, pkgformat=pkgformat, + architecture=architecture) #FIXME + app.file.save(packagePath(appId, architecture, tags_hash), pkgfile) app.save() + + for i in taglist: + # attach tags to app + app.tags.add(i) + diff --git a/store/osandarch.py b/store/osandarch.py index 7b833d3..5668b48 100644 --- a/store/osandarch.py +++ b/store/osandarch.py @@ -32,12 +32,14 @@ # check for file type here. # those are expected types -# ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.18, not stripped +# ELF 64-bit LSB executable, x86-64, version 1 (SYSV), dynamically linked, +# interpreter /lib64/ld-linux-x86-64.so.2, for GNU/Linux 2.6.18, not stripped # ELF 64-bit LSB shared object, x86-64, version 1 (GNU/Linux), dynamically linked, not stripped # ELF 64-bit LSB shared object, x86-64, version 1 (SYSV), dynamically linked, not stripped # Mach-O 64-bit x86_64 dynamically linked shared library # Mach-O 64-bit x86_64 executable -# Mach-O universal binary with 2 architectures: [x86_64: Mach-O 64-bit x86_64 bundle] [i386: Mach-O i386 bundle] [] +# Mach-O universal binary with 2 architectures: +# [x86_64: Mach-O 64-bit x86_64 bundle] [i386: Mach-O i386 bundle] [] # PE32+ executable (console) x86-64, for MS Windows # PE32+ executable (DLL) (console) x86-64, for MS Windows # PE32+ executable (DLL) (GUI) x86-64, for MS Windows @@ -45,21 +47,21 @@ import re -def parseMachO(str): # os, arch, bits, endianness - if " universal " in str: +def parseMachO(string_data): # os, arch, bits, endianness + if " universal " in string_data: # Universal binary - not supported raise Exception("Universal binaries are not supported in packages") os = "macOS" - arch = str.split(' ') + arch = string_data.split(' ') arch = arch[2] - bits = str.split(' ')[1].replace('-bit', '') + bits = string_data.split(' ')[1].replace('-bit', '') endianness = "little_endian" return [os, arch, bits, endianness] -def parsePE32(str): +def parsePE32(string_data): os = "Windows" - arch = str.split(',') + arch = string_data.split(',') arch = arch[0] # Take first part arch = arch.split(' ') arch = arch[-1] # Take last element @@ -72,12 +74,13 @@ def parsePE32(str): return [os, arch, bits, endianness] -def parseElfArch(str, architecture, bits): +def parseElfArch(string_data, architecture, bits): architecture = architecture.strip() if architecture.startswith("ARM"): if 'aarch64' in architecture: return 'arm64' - if 'armhf' in str: # this does not work for some reason - from_file() returns longer data than from_buffer() - needs fix + if 'armhf' in string_data: # this does not work for some reason - from_file() returns + # longer data than from_buffer() - needs fix return 'arm' # because qt does not report it directly elif architecture.startswith("Intel"): if '80386' in architecture: @@ -93,13 +96,13 @@ def parseElfArch(str, architecture, bits): return architecture.lower() -def parseElf(str): +def parseElf(string_data): os = "Linux" - arch = str.split(',') + arch = string_data.split(',') arch = arch[1] - bits = str.split(' ')[1].replace('-bit', '') - arch = parseElfArch(str, arch, bits) - endian = str.split(' ')[2].lower() + bits = string_data.split(' ')[1].replace('-bit', '') + arch = parseElfArch(string_data, arch, bits) + endian = string_data.split(' ')[2].lower() if endian == "msb": endianness = "big_endian" elif endian == "lsb": @@ -109,21 +112,21 @@ def parseElf(str): return [os, arch, bits, endianness] -def getOsArch(str): +def getOsArch(string_data): os = None arch = None bits = None endianness = None fmt = None - if str.startswith("ELF "): + if string_data.startswith("ELF "): fmt = "elf" - os, arch, bits, endianness = parseElf(str) - elif str.startswith("Mach-O "): + os, arch, bits, endianness = parseElf(string_data) + elif string_data.startswith("Mach-O "): fmt = "mach_o" - os, arch, bits, endianness = parseMachO(str) - elif str.startswith("PE32+ ") or str.startswith("PE32 "): + os, arch, bits, endianness = parseMachO(string_data) + elif string_data.startswith("PE32+ ") or string_data.startswith("PE32 "): fmt = "pe32" - os, arch, bits, endianness = parsePE32(str) + os, arch, bits, endianness = parsePE32(string_data) if arch: arch = arch.replace('-', '_') result = [os, arch, endianness, bits, fmt] @@ -133,9 +136,10 @@ def getOsArch(str): def normalizeArch(inputArch): """ - This function brings requested architecture to common form (currently just parses the bits part and turns it into 32/64) - Input string format is: arch-endianness-word_size-optional_ABI-kernelType - Output string format is: arch-endianness-word_size-kernelType + This function brings requested architecture to common form (currently just parses the bits + part and turns it into 32/64) + Input string format is: arch-endianness-word_size-optional_ABI-kernelType + Output string format is: arch-endianness-word_size-kernelType """ parts = inputArch.split('-') diff --git a/store/tags.py b/store/tags.py index aa73006..e906684 100644 --- a/store/tags.py +++ b/store/tags.py @@ -37,6 +37,10 @@ def validateTagVersion(version): + """ + Validates tag version, in string form + :type tag: str + """ for i in version: if not i.isalnum(): if not ((i == "_") or (i == ".")): @@ -45,7 +49,11 @@ def validateTagVersion(version): def validateTag(tag): - if len(tag) == 0: + """ + Validates tag (with version), in string form + :type tag: str + """ + if not tag: return False lst = tag.split(':') if len(lst) > 2: @@ -60,14 +68,21 @@ def validateTag(tag): class SoftwareTag: + """ + This class represents one tag instance. Tag is formatted in string form as two parts: + * tag itself, which is supposed to contain only alphanumeric symbols and _ + * optional tag version, which consists of any non-zero number of alphanumeric parts, + separated by dots. +""" + def __init__(self, tag): """ Takes tag and parses it. If it can't parse - raises exception of invalid value :type tag: str """ - if not ((type(tag) == str) or (type(tag) == unicode)): - raise (BaseException("Invalid input data-type")) + if not isinstance(tag, (str, unicode)): + raise BaseException("Invalid input data-type") if not validateTag(tag): - raise (BaseException("Malformed tag")) + raise BaseException("Malformed tag") tag_version = tag.split(':') self.tag = tag_version[0].lower() # No, this should be lowercase self.version = None if len(tag_version) == 1 else tag_version[1] @@ -78,37 +93,27 @@ def __repr__(self): def __str__(self): if self.version: return "%s:%s" % (self.tag, self.version) - else: - return self.tag + return self.tag def has_version(self): return self.version is not None def match(self, tag): # self is "on server", tag is "request" + """ + Does tag matching for tag string/tag list minimisation + :param tag: tags.SoftwareTag + :return: Returns true, if self is more specific (or equal) than tag + """ if self.tag == tag.tag: # Names are the same, that is good, matching versions now. if self.version == tag.version: return True - else: - if tag.version is None: - return True # qt in request, anything else on server - True - if self.version is not None and self.version.startswith(tag.version + "."): - return True - return False + if tag.version is None: + return True + if self.version is not None and self.version.startswith(tag.version + "."): + return True return False - def make_regex(self): - if self.version is None: - # versionless tag - temp_string = re.escape(self.tag) - regex = "(%s:[a-z0-9_.]*)|(%s)" % (temp_string, temp_string,) - else: - # tag with versions - temp_string = re.escape("%s:%s" % (self.tag, self.version)) - regex = "(%s\.[a-z0-9_.]*)|(%s)" % (temp_string, temp_string) - return regex - - class SoftwareTagList: def __init__(self): # dictionary of tags, key is - tag name @@ -116,7 +121,7 @@ def __init__(self): def __str__(self): lst = list() - for key, value in self.taglist.items(): + for _, value in self.taglist.items(): lst += [str(i) for i in value] lst.sort() return ",".join(lst) @@ -136,7 +141,8 @@ def parse(self, tag_string): def has_version(self, tag_name): if tag_name in self.taglist: - # This check is possible, because, when there is tag without version - it is the only tag in the list + # This check is possible, because, when there is tag without version - + # it is the only tag in the list if self.taglist[tag_name][0].has_version(): return True return False @@ -160,41 +166,13 @@ def append(self, tag): def is_empty(self): return len(self.taglist) == 0 - def make_regex(self): + def list(self): lst = list() - for key, value in self.taglist.items(): - regex = "(^|,)%s(,|$)" % "|".join([i.make_regex() for i in value]) - lst.append(regex) + for _, value in self.taglist.items(): + for i in value: + lst.append(i) return lst - def match_positive(self, taglist): - # checks that everything from tag list matches current tags - # Start checking with checking if all requested tags in taglist are present in self.taglist - for i in taglist.taglist: - if i not in self.taglist: - return False - # Now we need to check if versions are matching - for tag in taglist.taglist: - if not self.has_version(tag): - # If package tag accepts anything - it already matches, next please - continue - if taglist.has_version(tag) and not any(v1.match(v) for v in taglist[tag] for v1 in self[tag]): - return False - return True - - def match_negative(self, taglist): - # checks that nothing from taglist matches current tags - for i in taglist.taglist: - if i in self.taglist: - if (not taglist.has_version(i)) or (not self.has_version(i)): - return False - # Tag found, version list is present. check if it matches, if it does - check is failed - for version in taglist[i]: - for version1 in self[i]: - if version1.match(version): - return False - return True - def hash(self): # Looks like the list is sorted, but well... return hashlib.md5(str(self)).hexdigest() @@ -247,43 +225,6 @@ def test_not_empty_after_append(self): lst.append(SoftwareTag('qt')) self.assertFalse(lst.is_empty()) - def test_empty_matches_everything(self): - empty_list = SoftwareTagList() - test_list = SoftwareTagList() - test_list.append(SoftwareTag('qt')) - self.assertTrue(test_list.match_positive(empty_list)) - self.assertTrue(test_list.match_negative(empty_list)) - - def test_match_positive(self): - list_to_test = SoftwareTagList() - list_to_test.parse("qt:5.1,neptune,test:1,second_test") - matching_list = SoftwareTagList() - matching_list.parse("qt") - self.assertTrue(list_to_test.match_positive(matching_list)) - matching_list.parse("qt:5.1") - self.assertTrue(list_to_test.match_positive(matching_list)) - matching_list.parse("qt:5.1,qt:5.2,neptune:1") - self.assertTrue(list_to_test.match_positive(matching_list)) - matching_list.parse("qt:5.1,test:2") - self.assertFalse(list_to_test.match_positive(matching_list)) - matching_list.parse("qt:5.1.1") - self.assertFalse(list_to_test.match_positive(matching_list)) - - def test_match_negative(self): - list_to_test = SoftwareTagList() - list_to_test.parse("qt:5.1,neptune") - matching_list = SoftwareTagList() - matching_list.parse("qt") - self.assertFalse(list_to_test.match_negative(matching_list)) - matching_list.parse("qt:5.1") - self.assertFalse(list_to_test.match_negative(matching_list)) - matching_list.parse("qt:5.1,qt:5.2,neptune:1") - self.assertFalse(list_to_test.match_negative(matching_list)) - matching_list.parse("qt:5.1,qt:5.2") - self.assertFalse(list_to_test.match_negative(matching_list)) - matching_list.parse("test") - self.assertTrue(list_to_test.match_negative(matching_list)) - def test_append_invalid(self): lst = SoftwareTagList() with self.assertRaisesRegexp(BaseException, "Malformed tag"): diff --git a/store/utilities.py b/store/utilities.py index 1205f1b..74ffc98 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -43,38 +43,48 @@ from OpenSSL.crypto import load_pkcs12, FILETYPE_PEM, dump_privatekey, dump_certificate from M2Crypto import SMIME, BIO, X509 -from tags import SoftwareTagList, SoftwareTag -import osandarch +from store.tags import SoftwareTagList, SoftwareTag +import store.osandarch def makeTagList(pkgdata): + """Generates tag lists out of package data + First list - required tags, second list - conflicting tags + """ taglist = SoftwareTagList() + tagconflicts = SoftwareTagList() for fields in ('extra', 'extraSigned'): if fields in pkgdata['header']: if 'tags' in pkgdata['header'][fields]: for i in list(pkgdata['header'][fields]['tags']): # Fill tags list then add them taglist.append(SoftwareTag(i)) - return str(taglist) + if 'conflicts' in pkgdata['header'][fields]: + for i in list(pkgdata['header'][fields]['conflicts']): + tagconflicts.append(SoftwareTag(i)) + + tags_hash = str(taglist) + str(tagconflicts) + return taglist, tagconflicts, tags_hash def getRequestDictionary(request): if request.method == "POST": return request.POST - else: - return request.GET + return request.GET -def packagePath(appId = None, architecture = None, tags = None): +def packagePath(appId=None, architecture=None, tags=None): path = settings.MEDIA_ROOT + 'packages/' if tags is None: tags = "" if (appId is not None) and (architecture is not None): - path = path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_').replace(':','x3A').replace(',','x2C') + path = path + '_'.join([appId, architecture, tags]).replace('/', '_').\ + replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') return path -def iconPath(appId = None, architecture = None, tags = None): +def iconPath(appId=None, architecture=None, tags=None): path = settings.MEDIA_ROOT + 'icons/' if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/','_').replace('\\','_').replace(':','x3A').replace(',','x2C') + '.png' + return path + '_'.join([appId, architecture, tags]).replace('/', '_').\ + replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png' return path def writeTempIcon(appId, architecture, tags, icon): @@ -87,7 +97,8 @@ def writeTempIcon(appId, architecture, tags, icon): tempicon.close() return True, None except IOError as error: - return False, 'Validation error: could not write icon file to media directory: %s' % str(error) + return False, 'Validation error: could not write icon file to media directory: %s' % \ + str(error) def downloadPath(): return settings.MEDIA_ROOT + 'downloads/' @@ -300,7 +311,7 @@ def parsePackageMetadata(packageFile): fil.seek(0) #from_buffer instead of from_file filemagic = ms.from_file(fil.name) fil.close() - osarch = osandarch.getOsArch(filemagic) + osarch = store.osandarch.getOsArch(filemagic) if osarch: #[os, arch, endianness, bits, fmt] architecture = '-'.join(osarch[1:]) osset.add(osarch[0]) @@ -416,7 +427,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): certificates = [] for certFile in settings.APPSTORE_DEV_VERIFY_CA_CERTIFICATES: with open(certFile, 'rb') as cert: - certificates.append(cert.read()) + certificates.append(cert.read()) verifySignature(pkgdata['footer']['developerSignature'], pkgdata['rawDigest'], certificates) From 875289635bd5f15cc1610044864db54e188c21aa Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 22 Apr 2020 14:30:31 +0300 Subject: [PATCH 42/48] Documentation and virtual environment update to fix pkg-resources issue Change-Id: Ie56dc40934a5dfc502cee1336febf27fec212be3 Fixes: AUTOSUITE-1573 Reviewed-by: Egor Nemtsev (cherry picked from commit 8f45167406aa66a00bb0c28eb19a3eba267f1754) Reviewed-by: Nikolay Zamotaev --- doc/src/deployment-server.qdoc | 2 +- requirements.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 39d11dd..7698382 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -109,7 +109,7 @@ \section2 Set up the Server in a Virtual Environment Before you install the dependencies in the Python virtual environment, you must install the - \c{libffi-dev} package. Next, prepare the virtual environment: + \c{libffi-dev} and \c{python-pkg-resources} packages. Next, prepare the virtual environment: \code virtualenv -p python2.7 ./venv diff --git a/requirements.txt b/requirements.txt index c605671..92c71f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -pkg-resources==0.0.0 PyYAML django==1.11.27 django-common From 880d292c1be49bd60373ea1f7e6a14692e8645c6 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 8 Apr 2020 17:18:37 +0300 Subject: [PATCH 43/48] Documentation for package publishing process Change-Id: I717e88e26b38c2bcdbb1aa03eac0b29c6fbf32f8 Fixes: AUTOSUITE-1407 Reviewed-by: Svetlana Abramenkova (cherry-picked from commit 25fab1ce84f001044cc92b9abe26e1a627b528ce) Reviewed-by: Nikolay Zamotaev --- doc/src/deployment-server-package-upload.qdoc | 103 ++++++++++++++++++ .../commands/store-upload-package.py | 59 +++++----- 2 files changed, 132 insertions(+), 30 deletions(-) create mode 100644 doc/src/deployment-server-package-upload.qdoc diff --git a/doc/src/deployment-server-package-upload.qdoc b/doc/src/deployment-server-package-upload.qdoc new file mode 100644 index 0000000..b7176c8 --- /dev/null +++ b/doc/src/deployment-server-package-upload.qdoc @@ -0,0 +1,103 @@ +/**************************************************************************** +** +** Copyright (C) 2020 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-package-upload.html + \previouspage Qt Automotive Suite Deployment Server + \contentspage {Qt Automotive Suite} + + + \title Upload Packages to the Deployment Server + +Currently there are three possible ways to upload packages to the Deployment Server: using admin page +in web interface, using command line tool on the host where the server runs, and using REST API of the +server itself. + +\section1 Through Server Admin Page + +This was the first uploading method implemented. It uses django admin page, accessible by \c{/admin/} URL of +the Deployment Server. For Qt 5.14, the URL is \l{http://demoappsdeploy.qt.io:8514/admin/}. + +To add application: +\list + \li Navigate to the URL specified above + \li Login there as a user with administrative rights + \li Go to Apps section of the Store + \li Press \uicontrol{Add app} button. +\endlist + +At this point, there will be form to add an application, containing the following fields: file, vendor, category, +brief description, and description. + +\table + \header + \li Field + \li Field description + \row + \li File + \li Where the package file for upload is specified. + \row + \li Vendor + \li The package vendor. When package signing is enabled, vendor's certificate is used + to add package signature. When package signing is disabled, it is still required, but has no + real effect. + \row + \li Category + \li Category, in which a package will be shown on the deployment server. Currently it is + possible to select only one category, even if specification allows multiple categories to be + specified in a package file metadata. + \row + \li Description + \li {1,2} BriefDescription and Description fields are self-explanatory (and they will be + returned as-is in the related API requests). + \row + \li BriefDescription +\endtable + +After filling in the fields and pressing \uicontrol{save} button, the package will be added to the +deployment server or error message will be reported. + + +\section1 Through API + +It requires making POST request to \c{/upload} URL. Parameters are described on +\l{Qt Automotive Suite Deployment Server API Reference} page. + +\section1 Through Command Line Tool + +If it is possible to access the command line of the Deployment Server, this is preferable way to +upload packages. The tool is a \c{store-upload-package} implemented as a part of \c{manage.py} +django main program. The tool can be run with the following command line: +\code +manage.py store-upload-package --vendor --category [--description ] +\endcode +Vendor and category are specified as their name, not an ID in django database. Description and brief +description are filled with the same value by this tool. + + +*/ diff --git a/store/management/commands/store-upload-package.py b/store/management/commands/store-upload-package.py index 7af866b..fe7a93e 100644 --- a/store/management/commands/store-upload-package.py +++ b/store/management/commands/store-upload-package.py @@ -30,45 +30,42 @@ ## ############################################################################# -from optparse import make_option +import argparse + from django.core.management.base import BaseCommand, CommandError from django.core.files.base import ContentFile from store.models import Category, Vendor, savePackageFile from store.utilities import parseAndValidatePackageMetadata - class Command(BaseCommand): help = 'Uploads a package to the deployment server. This can be used for batch uploading.' - usage_string = 'Usage: manage.py store-upload-package --vendor --category ' \ - ' [--description ] ' - # FIXME: this doesn't work: - # see https://docs.djangoproject.com/en/1.8/howto/custom-management-commands/#django.core.management.BaseCommand.add_arguments - self.option_list = BaseCommand.option_list + ( - make_option('--vendor', - action='store', - type="string", - dest='vendor', - help='Vendor name'), - make_option('--category', - action='store', - type="string", - dest='category', - help='Category name'), - make_option('--description', - action='store', - type="string", - dest='description', - default="Empty description", - help='Short package description'), - ) + def add_arguments(self, parser): + parser.add_argument('--vendor', + action='store', + type=str, + dest='vendor', + help='Vendor name') + parser.add_argument('--category', + action='store', + type=str, + dest='category', + help='Category name') + parser.add_argument('--description', + action='store', + type=str, + dest='description', + default="Empty description", + help='Short package description') + parser.add_argument('package', + metavar='package', + type=argparse.FileType('rb'), + nargs=1, + help='package file to upload') + def handle(self, *args, **options): - if len(args) != 1: - raise CommandError(self.usage_string) - if (not options['vendor']) or (not options['category']): - raise CommandError(self.usage_string) category = Category.objects.all().filter(name__exact=options['category']) vendor = Vendor.objects.all().filter(name__exact=options['vendor']) if not category: @@ -77,8 +74,8 @@ def handle(self, *args, **options): raise CommandError('Non-existing vendor specified') try: - self.stdout.write('Parsing package %s' % args[0]) - packagefile = open(args[0], 'rb') + self.stdout.write('Parsing package %s' % options['package'][0].name) + packagefile = options['package'][0] pkgdata = parseAndValidatePackageMetadata(packagefile) self.stdout.write(' -> passed validation (internal name: %s)\n' % pkgdata['storeName']) except Exception as error: @@ -95,3 +92,5 @@ def handle(self, *args, **options): savePackageFile(pkgdata, ContentFile(packagefile.read()), package_metadata) except Exception as error: raise CommandError(error) + return 0 + From a2c8b2b94e119418bf204ff08dd9d108432243b8 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 24 Apr 2020 17:22:12 +0300 Subject: [PATCH 44/48] Documentation structure update Documentation updated for easier readability Change-Id: Iddaa22b974b60351e0b4237be5b3dee4cf1d1da7 Fixes: AUTOSUITE-1577 Reviewed-by: Evgeniy Zabotkin Reviewed-by: Svetlana Abramenkova (cherry picked from commit 6765a8536bdcce039991ac19cc800e0495f53c92) Task-number: AUTOSUITE-1617 Reviewed-by: Nikolay Zamotaev --- doc/qtautodeploymentserver-project.qdocconf | 1 + .../deployment-server-http-server-setup.qdoc | 6 +- doc/src/deployment-server-installation.qdoc | 136 ++++++++++++++++++ doc/src/deployment-server-package-upload.qdoc | 6 +- doc/src/deployment-server-reference.qdoc | 6 +- doc/src/deployment-server.qdoc | 106 ++------------ 6 files changed, 157 insertions(+), 104 deletions(-) create mode 100644 doc/src/deployment-server-installation.qdoc diff --git a/doc/qtautodeploymentserver-project.qdocconf b/doc/qtautodeploymentserver-project.qdocconf index 8625f13..258ef99 100644 --- a/doc/qtautodeploymentserver-project.qdocconf +++ b/doc/qtautodeploymentserver-project.qdocconf @@ -24,3 +24,4 @@ depends = qtautomotivesuite buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" navigation.homepage = "Qt Automotive Suite" +navigation.landingpage = "Qt Automotive Suite Deployment Server" diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc index a293931..6921362 100644 --- a/doc/src/deployment-server-http-server-setup.qdoc +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -28,8 +28,10 @@ /*! \page qtauto-deployment-server-http-server-setup.html - \previouspage Qt Automotive Suite Deployment Server - \contentspage Qt Automotive Suite + \previouspage Qt Automotive Suite Deployment Server API Reference + \nextpage Upload Packages to the Deployment Server + \contentspage {Qt Automotive Suite Deployment Server} + \startpage Qt Automotive Suite Deployment Server \title Set up a Production server with Apache, Lighttpd or Nginx diff --git a/doc/src/deployment-server-installation.qdoc b/doc/src/deployment-server-installation.qdoc new file mode 100644 index 0000000..4cb0517 --- /dev/null +++ b/doc/src/deployment-server-installation.qdoc @@ -0,0 +1,136 @@ +/**************************************************************************** +** +** Copyright (C) 2019 Luxoft Sweden AB +** Copyright (C) 2018 Pelagicore AG +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the documentation of the Qt Auto Deployment Server. +** +** $QT_BEGIN_LICENSE:FDL-QTAS$ +** Commercial License Usage +** Licensees holding valid commercial Qt Automotive Suite licenses may use +** this file in accordance with the commercial license agreement provided +** with the Software or, alternatively, in accordance with the terms +** contained in a written agreement between you and The Qt Company. For +** licensing terms and conditions see https://www.qt.io/terms-conditions. +** For further information use the contact form at https://www.qt.io/contact-us. +** +** GNU Free Documentation License Usage +** Alternatively, this file may be used under the terms of the GNU Free +** Documentation License version 1.3 as published by the Free Software +** Foundation and appearing in the file included in the packaging of +** this file. Please review the following information to ensure +** the GNU Free Documentation License version 1.3 requirements +** will be met: https://www.gnu.org/licenses/fdl-1.3.html. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +/*! + \page qtauto-deployment-server-installation.html + \contentspage {Qt Automotive Suite Deployment Server} + \previouspage Qt Automotive Suite Deployment Server + \nextpage Qt Automotive Suite Deployment Server API Reference + \startpage Qt Automotive Suite Deployment Server + + \title Qt Automotive Suite Deployment Server Installation + + \section1 Set up the Server in a Virtual Environment + + Before you install the dependencies in the Python virtual environment, you must install the + \c{libffi-dev}, \c{python-pkg-resources}, \c{python-dev} and \c{libssl-dev} packages. Next, + prepare the virtual environment: + + \code + virtualenv -p python2.7 ./venv + ./venv/bin/pip install -r requirements.txt + \endcode + + Make sure to adapt the \c{APPSTORE_*} settings in \c{appstore/settings.py} to your environment, + before you run the server. + + One setting group to note is category icon resizing and decolorization settings. They are: + \c{ICON_SIZE_X}, \c{ICON_SIZE_Y} and \c{ICON_DECOLOR}. \c{ICON_DECOLOR} is False by default, and it + regulates stripping the category icon of color and alpha channel and the subsequent application + of the same image as an alpha channel instead. + + \c{ICON_SIZE_X} and \c{ICON_SIZE_Y} are the maximum icon sizes, icons are scaled in such a way, + as not to exceed this size, while keeping the aspect ratio of the icon. Default values are 50x50, + so you should be aware that any uploaded icon will be limited to this size, unless the server is + reconfigured. + + Since package downloads are done via temporary files, you need to setup a cron-job to remove + these temporary files periodically. The cron-job should be triggered every + \c{settings.APPSTORE_DOWNLOAD_EXPIRY/2} minutes; it needs to run: + + \code + ./manage.py expire-downloads + \endcode + + \section1 Activate the Python Virtual Environment + + Before you run \c manage.py, source the activation script on the console where you will be using it. + + \code + . ./venv/bin/activate + \endcode + + This activation is done once per session only: + + + \section1 Prepare Your Databases on the Server + + Before you run the server, first, you must prepare the databases. Also, you need to create an administrative user. + + \code + ./manage.py makemigrations + ./manage.py migrate + ./manage.py createsuperuser + \endcode + + Make sure that the server has a user added for Neptune 3 UI's Downloads App. Currently the username and + password are hardcoded in \c apps/com.pelagicore.downloads/stores/ServerConfig.qml:78. + + + \section1 Start the Server + + To start the server, run the following command in your terminal: + + \code + ./manage.py runserver 0.0.0.0:8080 + \endcode + + This command starts the server on port 8080, and is reachable by anyone. You can modify the + listening address to another address that suits your use case. + + \section1 Maintain the Server + + \list + \li Clean up the downloads directory: + \code + ./manage.py expire-downloads + \endcode + + This command removes all files from the downloads directory, that are older than + \c{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. Ideally, this command should be run via a cron-job. + + \li Manually verify a package for upload: + + \code + ./manage.py verify-upload-package + \endcode + + This command verifies if \c{} is a valid package that can be uploaded to the Downloads + app. + + \li Manually add a store signature to a package: + + \code + ./manage.py store-sign-package [device id] + \endcode + + This command first verifies the \c{}. If this verification succeeds, \c{} + is copied over to \c{} and a store signature is added. The optional \c{[device id]} + parameter locks the generated package to the device with this ID. + \endlist +*/ diff --git a/doc/src/deployment-server-package-upload.qdoc b/doc/src/deployment-server-package-upload.qdoc index b7176c8..dc5b7b4 100644 --- a/doc/src/deployment-server-package-upload.qdoc +++ b/doc/src/deployment-server-package-upload.qdoc @@ -28,8 +28,10 @@ /*! \page qtauto-deployment-server-package-upload.html - \previouspage Qt Automotive Suite Deployment Server - \contentspage {Qt Automotive Suite} + \previouspage Set up a Production server with Apache, Lighttpd or Nginx + \nextpage Qt Automotive Suite Deployment Server + \contentspage {Qt Automotive Suite Deployment Server} + \startpage Qt Automotive Suite Deployment Server \title Upload Packages to the Deployment Server diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index a8903de..cbc4cf3 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -28,8 +28,10 @@ /*! \page qtauto-deployment-server-reference.html - \previouspage Qt Automotive Suite Deployment Server - \contentspage Qt Automotive Suite + \previouspage Qt Automotive Suite Deployment Server Installation + \nextpage Set up a Production server with Apache, Lighttpd or Nginx + \contentspage {Qt Automotive Suite Deployment Server} + \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server API Reference diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 7698382..bcf2422 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -29,7 +29,8 @@ /*! \page qtauto-deployment-server-index.html \contentspage {Qt Automotive Suite} - \nextpage Qt Automotive Suite Deployment Server API Reference + \nextpage Qt Automotive Suite Deployment Server Installation + \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server @@ -104,103 +105,12 @@ This command creates package and adds \c{platform} tag with \c{desktop} value to it. For reference see the \l{Packager} section of \l{Qt Application Manager} documentation. - \section1 Installation - - \section2 Set up the Server in a Virtual Environment - - Before you install the dependencies in the Python virtual environment, you must install the - \c{libffi-dev} and \c{python-pkg-resources} packages. Next, prepare the virtual environment: - - \code - virtualenv -p python2.7 ./venv - ./venv/bin/pip install -r requirements.txt - \endcode - - Make sure to adapt the \c{APPSTORE_*} settings in \c{appstore/settings.py} to your environment, - before you run the server. - - One setting group to note is category icon resizing and decolorization settings. They are: - \c{ICON_SIZE_X}, \c{ICON_SIZE_Y} and \c{ICON_DECOLOR}. \c{ICON_DECOLOR} is False by default, and it - regulates stripping the category icon of color and alpha channel and the subsequent application - of the same image as an alpha channel instead. - - \c{ICON_SIZE_X} and \c{ICON_SIZE_Y} are the maximum icon sizes, icons are scaled in such a way, - as not to exceed this size, while keeping the aspect ratio of the icon. Default values are 50x50, - so you should be aware that any uploaded icon will be limited to this size, unless the server is - reconfigured. - - Since package downloads are done via temporary files, you need to setup a cron-job to remove - these temporary files periodically. The cron-job should be triggered every - \c{settings.APPSTORE_DOWNLOAD_EXPIRY/2} minutes; it needs to run: - - \code - ./manage.py expire-downloads - \endcode - - \section2 Activate the Python Virtual Environment - - Before you run \c manage.py, source the activation script on the console where you will be using it. - - \code - . ./venv/bin/activate - \endcode - - This activation is done once per session only: - - - \section2 Prepare Your Databases on the Server - - Before you run the server, first, you must prepare the databases. Also, you need to create an administrative user. - - \code - ./manage.py makemigrations - ./manage.py migrate - ./manage.py createsuperuser - \endcode - - Make sure that the server has a user added for Neptune 3 UI's Downloads App. Currently the username and - password are hardcoded in \c apps/com.pelagicore.downloads/stores/ServerConfig.qml:78. - - - \section2 Start the Server - - To start the server, run the following command in your terminal: - - \code - ./manage.py runserver 0.0.0.0:8080 - \endcode - - This command starts the server on port 8080, and is reachable by anyone. You can modify the - listening address to another address that suits your use case. - - \section2 Maintain the Server - + \section1 References and procedures \list - \li Clean up the downloads directory: - \code - ./manage.py expire-downloads - \endcode - - This command removes all files from the downloads directory, that are older than - \c{settings.APPSTORE_DOWNLOAD_EXPIRY} minutes. Ideally, this command should be run via a cron-job. - - \li Manually verify a package for upload: - - \code - ./manage.py verify-upload-package - \endcode - - This command verifies if \c{} is a valid package that can be uploaded to the Downloads - app. - - \li Manually add a store signature to a package: - - \code - ./manage.py store-sign-package [device id] - \endcode - - This command first verifies the \c{}. If this verification succeeds, \c{} - is copied over to \c{} and a store signature is added. The optional \c{[device id]} - parameter locks the generated package to the device with this ID. + \li \l{Qt Automotive Suite Deployment Server Installation} + \li \l{Qt Automotive Suite Deployment Server API Reference} + \li \l{Set up a Production server with Apache, Lighttpd or Nginx} + \li \l{Upload Packages to the Deployment Server} \endlist + */ From 84a6bc29d890416b2f7f42edcbce4cca4a7ea565 Mon Sep 17 00:00:00 2001 From: Evgeniy Zabotkin Date: Wed, 29 Apr 2020 15:12:48 +0300 Subject: [PATCH 45/48] Deployment Server Documentation Update Change-Id: Ic1559fd79e1de205ea59cab25681f44c66a9a530 Reviewed-by: Svetlana Abramenkova Task-number: AUTOSUITE-1617 (cherry picked from commit 76219d7c94684aba09504eb2138e9e11444f8825) Reviewed-by: Nikolay Zamotaev --- .../deployment-server-http-server-setup.qdoc | 18 +++++++++--------- doc/src/deployment-server-installation.qdoc | 14 +++++++------- doc/src/deployment-server-reference.qdoc | 6 +++--- doc/src/deployment-server.qdoc | 12 ++++++------ 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc index 6921362..a155b3d 100644 --- a/doc/src/deployment-server-http-server-setup.qdoc +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -40,8 +40,8 @@ The Deployment Server can be set up in combination with a regular web server: Ap Nginx. This web server can be used to reduce the number of open ports or to add a layer of SSL encryption. -In the Django version that is used for the deployment server, there are two possible setups for - running with another web server: +There are two possible setups for running with another web server in the Django version used for +the Deployment Server: \list \li reverse proxy setup - which we describe here \li WSGI setup - for brevity, this setup is left out of scope, as it is similar in functionality @@ -108,7 +108,7 @@ $HTTP["url"] =~ "^//" { \endcode Where \e{} is the same as in \e{settings.py} file, and \e{} is the port on which the -deployment server instance is running. +Deployment Server instance is running. Finally, reload the Lighttpd server's configuration. @@ -125,17 +125,17 @@ The code that needs to be added is: \endcode Where \e{} is the same as in \e{settings.py} file, and \e{} is the port on which the -deployment server instance is running. +Deployment Server instance is running. -\section1 Serving static files outside of django app +\section1 Serving Static Files Outside of Django App -It is possible to speed up working of the admin pages of deployment server by serving static files -separately. In order to achieve that, webserver should be configured to serve \e{static/} subdirectory of -deployment server installation as \e{static/} subdirectory of the instance. (If deployment server is +It is possible to speed up working of the admin pages of the Deployment Server by serving static files +separately. In order to achieve that, web server should be configured to serve \e{static/} subdirectory of +the Deployment Server installation as \e{static/} subdirectory of the instance. (If the Deployment Server is hosted as \e{http://deployment.server.name/}, then \e{static/} directory in the sources should be redirected to \e{http://deployment.server.name/static/}, while bypassing the django app.) -After configuring that, static files collection should be performed with this command: +After configuring that, static files collection should be performed using the following command: \code ./venv/bin/python manage.py collect static diff --git a/doc/src/deployment-server-installation.qdoc b/doc/src/deployment-server-installation.qdoc index 4cb0517..7937992 100644 --- a/doc/src/deployment-server-installation.qdoc +++ b/doc/src/deployment-server-installation.qdoc @@ -37,8 +37,8 @@ \section1 Set up the Server in a Virtual Environment - Before you install the dependencies in the Python virtual environment, you must install the - \c{libffi-dev}, \c{python-pkg-resources}, \c{python-dev} and \c{libssl-dev} packages. Next, + Before you install the dependencies in the Python virtual environment, you need to install the + \c{libffi-dev}, \c{python-pkg-resources}, \c{python-dev} and \c{libssl-dev} packages. Then, prepare the virtual environment: \code @@ -50,7 +50,7 @@ before you run the server. One setting group to note is category icon resizing and decolorization settings. They are: - \c{ICON_SIZE_X}, \c{ICON_SIZE_Y} and \c{ICON_DECOLOR}. \c{ICON_DECOLOR} is False by default, and it + \c{ICON_SIZE_X}, \c{ICON_SIZE_Y}, and \c{ICON_DECOLOR}. \c{ICON_DECOLOR} is False by default, and it regulates stripping the category icon of color and alpha channel and the subsequent application of the same image as an alpha channel instead. @@ -59,7 +59,7 @@ so you should be aware that any uploaded icon will be limited to this size, unless the server is reconfigured. - Since package downloads are done via temporary files, you need to setup a cron-job to remove + Since package downloads are done through temporary files, you need to setup a cron-job to remove these temporary files periodically. The cron-job should be triggered every \c{settings.APPSTORE_DOWNLOAD_EXPIRY/2} minutes; it needs to run: @@ -80,7 +80,7 @@ \section1 Prepare Your Databases on the Server - Before you run the server, first, you must prepare the databases. Also, you need to create an administrative user. + Before you run the server, first, you need to prepare the databases. Also, you need to create an administrative user. \code ./manage.py makemigrations @@ -100,8 +100,8 @@ ./manage.py runserver 0.0.0.0:8080 \endcode - This command starts the server on port 8080, and is reachable by anyone. You can modify the - listening address to another address that suits your use case. + This command starts the server on port 8080, and is reachable by anyone. You can replace the + listening address with another one that suits your use case. \section1 Maintain the Server diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index cbc4cf3..d4be3af 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -98,7 +98,7 @@ \section2 login - Logs onto the deployment server with the given username and password. + Logs onto the Deployment Server with the given username and password. Either an IMEI or a unique hardware identifier, such as a MAC address, must be provided. This call is necessary to be able to download apps. \table @@ -136,7 +136,7 @@ \endtable \section2 logout - Logs out the currently logged-in user from the deployment server. + Logs out the currently logged-in user from the Deployment Server. Returns a JSON object with the following fields and values. \table @@ -296,7 +296,7 @@ \row \li failed \li - \li An error has occurred, dheck the error field for more information. + \li An error has occurred, check the error field for more information. \row \li error \li Text. diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index bcf2422..50f2bee 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -41,21 +41,21 @@ that are available for installation by a Qt Application Manager instance, running on a target device connected to a network. The UI counterpart for the Deployment Server is the Downloads app in the Neptune 3 UI. Together, the Deployment Server and the Downloads app enable you to install - different apps that are available in the server via network. + different apps available on the server through network. The key scenario for these components is to install apps during the integration testing process. Additionally, the code can also be used as a reference implementation for a fully-featured server, and a new Downloads app on the target device for production. - The Deployment Server works with the Application Installer in the Qt Application Manager and + The Deployment Server works with the Application Installer in \l{Qt Application Manager} and acts as an installation source for \c{http://} and \c{https://} schemes. In addition to application packages, the Deployment Server also hosts meta information about each package, - that is used by the Downloads app to visualize the choices available for the user to select. + that is used by Downloads app to visualize the choices available for a user to select. This meta information is stored in the form of tags and other information in the package header, according to the Qt Application Manager’s package format. When a package is uploaded - to the server, the package header is parsed, associated with that package, and then sent to a - Downloads app, that queries for a list of available apps. Using this information, a Downloads - app can inform users about apps that are available and even hide those that are not compatible + to the server, the package header is parsed, associated with that package, and then sent to the + Downloads app, that queries for a list of available apps. Using this information, the Downloads + app can inform users about the available apps and even hide those that are not compatible with the target installation. The figure below illustrates this installation scenario. \image deployment-server-installation-handshake.png "Install an App via the Deployment Server" From 561c1525e11f3561f6afe6817a77198cbc363680 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Fri, 11 Sep 2020 15:51:57 +0300 Subject: [PATCH 46/48] Package verification fix Package verification was not checking if package tags were valid Change-Id: I0b788b59179fc18370ac3fb619f22126358bc0c8 Fixes: AUTOSUITE-1615 Reviewed-by: Egor Nemtsev --- store/api.py | 2 ++ store/utilities.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/store/api.py b/store/api.py index 75e730f..9444b0b 100644 --- a/store/api.py +++ b/store/api.py @@ -178,6 +178,8 @@ def upload(request): except Exception as error: status = str(error) + except BaseException as error: + status = str(error) return JsonResponse({'status': status}) diff --git a/store/utilities.py b/store/utilities.py index 74ffc98..06151bb 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -413,6 +413,11 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): if not name: raise Exception('could not deduce a suitable package name from the info part') + try: + _, _, _ = makeTagList(pkgdata) + except BaseException as error: + raise Exception(str(error)) + pkgdata['storeName'] = name if pkgdata['digest'] != pkgdata['footer']['digest']: From 7259d1a839a698e68bc4a7020a63d2aca79a5ec6 Mon Sep 17 00:00:00 2001 From: Nikolay Zamotaev Date: Wed, 11 Nov 2020 19:00:50 +0300 Subject: [PATCH 47/48] Fix for broken verify-upload-package and store-sign-package commands Task-number: AUTOSUITE-1631 Change-Id: I7c17bc2a34ead2f094a2b3d28315191e80912a98 Pick-to: 5.15 Reviewed-by: Grigorii Zimin --- .../management/commands/store-sign-package.py | 44 ++++++++++++++----- .../commands/verify-upload-package.py | 17 ++++--- 2 files changed, 44 insertions(+), 17 deletions(-) diff --git a/store/management/commands/store-sign-package.py b/store/management/commands/store-sign-package.py index cf51670..ade1546 100644 --- a/store/management/commands/store-sign-package.py +++ b/store/management/commands/store-sign-package.py @@ -30,31 +30,51 @@ ## ############################################################################# +import argparse + from django.core.management.base import BaseCommand, CommandError -from django.conf import settings from store.utilities import parseAndValidatePackageMetadata, addSignatureToPackage class Command(BaseCommand): help = 'Adds a store signature to the package' + def add_arguments(self, parser): + parser.add_argument('source-package', + metavar='sourcepackage', + type=argparse.FileType('rb'), + nargs=1, + help='package file to sign') + parser.add_argument('destination-package', + metavar='destinationpackage', + type=str, + nargs=1, + help='signed package file destination') + parser.add_argument('device ID', + metavar='deviceID', + type=str, + nargs='?', + default="", + help='device ID') + def handle(self, *args, **options): - if 2 > len(args) > 3: - raise CommandError('Usage: manage.py store-sign-package [device id]') + if not options["source-package"]: + raise CommandError('Usage: manage.py store-sign-package \ + [device id]') - sourcePackage = args[0] - destinationPackage = args[1] - deviceId = args[2] if len(args) == 3 else '' + source_package = options["source-package"] + destination_package = options["destination-package"][0] + device_id = options["device ID"] try: - self.stdout.write('Parsing package %s' % sourcePackage) - packageFile = open(sourcePackage, 'rb') - pkgdata = parseAndValidatePackageMetadata(packageFile) + self.stdout.write('Parsing package %s' % source_package[0].name) + package_file = source_package[0] + pkgdata = parseAndValidatePackageMetadata(package_file) self.stdout.write(' -> passed validation (internal name: %s)\n' % pkgdata['storeName']) - self.stdout.write('Adding signature to package %s' % destinationPackage) - addSignatureToPackage(sourcePackage, destinationPackage, pkgdata['rawDigest'], - deviceId, pkgdata['packageFormat']['formatVersion']) + self.stdout.write('Adding signature to package %s' % destination_package) + addSignatureToPackage(source_package[0].name, destination_package, pkgdata['rawDigest'], + device_id, pkgdata['packageFormat']['formatVersion']) self.stdout.write(' -> finished') except Exception as error: diff --git a/store/management/commands/verify-upload-package.py b/store/management/commands/verify-upload-package.py index 59df116..298e3aa 100644 --- a/store/management/commands/verify-upload-package.py +++ b/store/management/commands/verify-upload-package.py @@ -30,7 +30,7 @@ ## ############################################################################# -import sys +import argparse from django.core.management.base import BaseCommand, CommandError @@ -39,14 +39,21 @@ class Command(BaseCommand): help = 'Checks if packages are valid for store upload' + def add_arguments(self, parser): + parser.add_argument('package', + metavar='package', + type=argparse.FileType('rb'), + nargs=1, + help='package file to verify') + def handle(self, *args, **options): - if len(args) != 1: + if not options["package"]: raise CommandError('Usage: manage.py verify-upload-package ') try: - self.stdout.write('Parsing package %s' % args[0]) - packageFile = open(args[0], 'rb') - pkgdata = parseAndValidatePackageMetadata(packageFile) + self.stdout.write('Parsing package %s' % options['package'][0].name) + package_file = options['package'][0] + pkgdata = parseAndValidatePackageMetadata(package_file) self.stdout.write(' -> passed validation (internal name: %s)\n' % pkgdata['storeName']) From b3665620377a06f7b7a012e2ae7b69d222fae435 Mon Sep 17 00:00:00 2001 From: Robert Griebl Date: Fri, 29 Jul 2022 23:18:37 +0200 Subject: [PATCH 48/48] Port to python 3 and django 4.0.6 PLEASE NOTE: This project is not maintained anymore. It was ported to a Qt 6 cmake setup and a more modern Django and Python version to at least keep it usable for legacy projects. For non-production use-cases, please switch to the new appman-package-server available in the Qt Application Manager starting with version 6.7. Task-number: AUTOSUITE-1368 Change-Id: Idc4f2490a2a4399c03fce761250f4b5ac2612a45 Reviewed-by: Dominik Holland --- .cmake.conf | 1 + .gitignore | 8 +- CMakeLists.txt | 36 +++ Dockerfile | 45 +++ README.md | 259 ------------------ appstore/settings.py | 45 +-- appstore/urls.py | 30 +- doc/QtAutoDeploymentServerDoc | 3 + doc/qtautodeploymentserver-project.qdocconf | 13 +- .../deployment-server-http-server-setup.qdoc | 5 +- doc/src/deployment-server-installation.qdoc | 14 +- doc/src/deployment-server-package-upload.qdoc | 6 +- doc/src/deployment-server-reference.qdoc | 4 +- doc/src/deployment-server.qdoc | 10 +- docker-entrypoint.sh | 4 + docker-manage.sh | 32 +++ manage.py | 2 +- qtauto-deployment-server.pro | 13 - requirements.txt | 6 +- store/admin.py | 43 ++- store/api.py | 43 +-- store/authdecorators.py | 4 +- store/management/commands/expire-downloads.py | 4 +- store/migrations/0001_initial.py | 9 +- store/migrations/0002_alter_category_order.py | 18 ++ store/models.py | 14 +- store/tags.py | 20 +- store/utilities.py | 77 +++--- tests/tests.pro | 1 - 29 files changed, 319 insertions(+), 450 deletions(-) create mode 100644 .cmake.conf create mode 100644 CMakeLists.txt create mode 100644 Dockerfile delete mode 100644 README.md create mode 100644 doc/QtAutoDeploymentServerDoc create mode 100755 docker-entrypoint.sh create mode 100755 docker-manage.sh delete mode 100644 qtauto-deployment-server.pro create mode 100644 store/migrations/0002_alter_category_order.py delete mode 100644 tests/tests.pro diff --git a/.cmake.conf b/.cmake.conf new file mode 100644 index 0000000..56186d7 --- /dev/null +++ b/.cmake.conf @@ -0,0 +1 @@ +set(QT_REPO_MODULE_VERSION "6.7.0") diff --git a/.gitignore b/.gitignore index cd45596..dfc68b5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,3 @@ *~ *.pyc -db.sqlite3 -media/ -static/ -certificates/ -.idea/* -venv/* -static/* +data/* diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..d3c8839 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,36 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +cmake_minimum_required(VERSION 3.16) + +include(.cmake.conf) +project(QtAutoDeploymentServer + VERSION "${QT_REPO_MODULE_VERSION}" + DESCRIPTION "QtAuto deployment server" + HOMEPAGE_URL "https://qt.io/" + LANGUAGES CXX C +) + +find_package(Qt6 ${PROJECT_VERSION} CONFIG REQUIRED COMPONENTS BuildInternals Core Network) + +add_custom_target(create_docker + COMMAND docker build -t qtauto-deployment-server . + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} + VERBATIM +) + +qt_build_repo() + + +if(QT_BUILD_ONLINE_DOCS) + set(DOC_CONF "doc/online/qtautodeploymentserver.qdocconf") +else() + set(DOC_CONF "doc/qtautodeploymentserver.qdocconf") +endif() + +file(GLOB_RECURSE allDocFiles "doc/*.qdoc" "doc/*.png" "doc/*.qdocconf") +add_custom_target(Documentation SOURCES ${allDocFiles}) +qt_internal_add_docs(Documentation ${DOC_CONF}) + +# Add tool dependencies that were deferred by qt_internal_add_docs. +qt_internal_add_deferred_dependencies() diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..1c0cbc5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,45 @@ + +FROM debian:bullseye-slim +MAINTAINER Robert Griebl + +ENV LC_ALL="C.UTF-8" + +RUN apt-get update && apt-get install -y --no-install-recommends \ + python3-pip \ + python3-magic \ + python3-m2crypto + +RUN rm -rf /var/lib/apt/lists/* + +COPY requirements.txt / +RUN pip3 install -r requirements.txt + +RUN mkdir /server +COPY manage.py /server +RUN chmod 755 ./server/manage.py +COPY appstore/ /server/appstore +COPY store/ /server/store + +RUN mkdir /data +VOLUME /data + +ENV APPSTORE_DATA_PATH=/data +ENV APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE /data/certificates/store.p12 +ENV APPSTORE_DEV_VERIFY_CA_CERTIFICATES /data/certificates/ca.crt,/data/certificates/devca.crt + +# You can also set these environment variables: +## ENV APPSTORE_PLATFORM_ID NEPTUNE3 +## ENV APPSTORE_PLATFORM_VERSION 2 +## ENV APPSTORE_DOWNLOAD_EXPIRY 10 +## ENV APPSTORE_BIND_TO_DEVICE_ID 1 +## ENV APPSTORE_NO_SECURITY 1 +## ENV APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE certificates/store.p12 +## ENV APPSTORE_STORE_SIGN_PKCS12_PASSWORD password +## ENV APPSTORE_DEV_VERIFY_CA_CERTIFICATES certificates/ca.crt,certificates/devca.crt + + +COPY docker-entrypoint.sh / +ENTRYPOINT [ "/docker-entrypoint.sh" ] +CMD [ "runserver", "0.0.0.0:8080" ] + +EXPOSE 8080 diff --git a/README.md b/README.md deleted file mode 100644 index 28a473e..0000000 --- a/README.md +++ /dev/null @@ -1,259 +0,0 @@ -This is a PoC deployment server, which can be used together with -the Neptune IVI UI and the Luxoft Application Manager. - -**This is a development server only - do NOT use in production.** - -Architecture -============ - -Setting up the server in virtualenv: - -virtualenv ./venv -./venv/bin/pip install -r requirements.txt - -(libffi-dev is also needed) - - -The server is based on Python/Django. -The reference platform is Debian Jessie and the packages needed there are: - - * python (2.7.9) - * python-yaml (3.11) - * python-django (1.7.9) - * python-django-common (1.7.9) - * python-openssl (0.14) - * python-m2crypto (0.21) - -Before running the server, make sure to adapt the `APPSTORE_*` settings in -`appstore/settings.py` to your environment. - -Since package downloads are done via temporary files, you need to setup -a cron-job to cleanup these temporary files every now and then. The job -should be triggerd every (`settings.APPSTORE_DOWNLOAD_EXPIRY` / 2) minutes -and it just needs to execute: - -``` - ./manage.py expire-downloads -``` - -Commands -======== - -* Running the server: - ``` - ./manage.py runserver 0.0.0.0:8080 - ``` - will start the server on port 8080, reachable for anyone. You can tweak - the listening address to whatever fits your needs. - -* Cleaning up the downloads directory: - ``` - ./manage.py expire-downloads - ``` - will remove all files from the downloads/ directory, that are older than - `settings.APPSTORE_DOWNLOAD_EXPIRY` minutes. - This should be called from a cron-job (see above). - -* Manually verifying a package for upload: - ``` - ./manage.py verify-upload-package - ``` - will tell you if `` is a valid package that can be uploaded to - the store. - -* Manually adding a store signature to a package: - ``` - ./manage.py store-sign-package [device id] - ``` - will first verify ``. If this succeeds, it will copy `` - to `` and add a store signature. The optional `[device id]` - parameter will lock the generated package to the device with this id. - -HTTP API -======== - -The deployment server exposes a HTTP API to the world. Arguments to the -functions need to be provided using the HTTP GET syntax. The returned data -will be JSON, PNG or text, depending on the function - -Basic workflow: - -1. Send a `"hello"` request to the server to get the current status and check - whether your platform is compatible with this server instance: - ``` - http:///hello?platform=AM&version=1 - ``` - Returns: - ``` - { "status": "ok" } - ``` - -2. Login as user `'user'` with password `'pass'`: - ``` - http:///login?username=user&password=pass - ``` - Returns: - ``` - { "status": "ok" } - ``` - -3. List all applications - ``` - http:///app/list - ``` - Returns: - ``` - [{ "category": "Entertainment", - "name": "Nice App", - "vendor": "Pelagicore", - "briefDescription": "Nice App is a really nice app.", - "category_id": 4, - "id": "com.pelagicore.niceapp"}, - ... - ] - ``` - -4. Request a download for a App: - ``` - http:///app/purchase?device_id=12345&id=com.pelagicore.niceapp - ``` - Returns: - ``` - { "status": "ok", - "url": "http:///app/download/com.pelagicore.niceapp.2.npkg", - "expiresIn": 600 - } - ``` - -5. Use the `'url'` provided in step 4 to download the application within - `'expiresIn'` seconds. - - -API Reference -============= - -## hello -Checks whether you are using the right Platform and the right API to communicate with the Server. - -| Parameter | Description | -| ---------- | ----------- | -| `platform` | The platform the client is running on, this sets the architecture of the packages you get. (see `settings.APPSTORE_PLATFORM`) | -| `version` | The Deployment Server HTTP API version you are using to communicate with the server. (see `settings.APPSTORE_VERSION`) | -| `require_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. Only applications containing any of specified tags should be listed. Tags must be alphanumeric. | -| `conflicts_tag` | Optional parameter for filtering packages by tags. Receives coma-separated list of tags. No application containing any of the tags should be listed. Tags must be alphanumeric. | -| `architecture` | Optional parameter for filtering packages by architecture. Receives cpu architecture. If architecture is not specified, only packages showing 'All' architecture are listed. | - -Returns a JSON object: - -| JSON field | Value | Description | -| ---------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `maintenance` | The Server is in maintenance mode and can't be used at the moment. | -| | `incompatible-platform` | You are using an incompatible Platform. | -| | `incompatible-version` | You are using an incompatible Version of the API. | -| | `malformed-tag` | Tag had wrong format, was not alphanumeric or could not be parsed. | - -## login -Does a login on the Server with the given username and password. Either a imei or a mac must be provided. This call is needed for downloading apps. - -| Parameter | Description | -| ---------- | ----------- | -| `username` | The username. | -| `password` | The password for the given username | - -Returns a JSON object: - -| JSON field | Value | Description | -| ---------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `missing-credentials` | Forgot to provided username and/or password. | -| | `account-disabled` | The account is disabled. | -| | `authentication-failed` | Failed to authenticate with given username and password. | - - -## logout -Does a logout on the Server for the currently logged in user. - -Returns a JSON object: - -| JSON field | Value | Description | -| ---------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `failed` | Not logged in. | - -## app/list -Lists all apps. The returned List can be filtered by using the category_id and the filter argument. - -| Parameter | Description | -| ------------- | ----------- | -| `category_id` | Only lists apps, which are in the category with this id. | -| `filter` | Only lists apps, whose name matches the filter. | - -Returns a JSON array (not an object!). Each field is a JSON object: - -| JSON field | Description | -| ------------------ | ----------- | -| `id` | The unique id of the application | -| `name` | The name of the application | -| `vendor` | The name of the vendor of this application -| `briedDescription` | A short (one line) description of the application -| `category` | The name of the category the application is in -| `category_id` | The id of the category the application is in - -## app/icon - Returns an icon for the given application id. - -| Parameter | Description | -| ---------- | ----------- | -| `id` | The application id | - - Returns a PNG image or a 404 error - - -## app/description -Returns a description for the given application id. - -| Parameter | Description | -| ---------- | ----------- | -| `id` | The application id | - -Returns text - either HTML or plain - - -## app/purchase -Returns an url which can be used for downloading the requested application for -certain period of time (configurable in the settings) - -| Parameter | Description | -| ----------- | ----------- | -| `device_id` | The unique device id of the client hardware. | -| `id` | The application Id. | - -Returns a JSON object: - -| JSON field | Value | Description | -| ----------- | --------- | ----------- | -| `status` | `ok` | Successfull. | -| | `failed` | Something went wrong. See the `error` field for more information. | -| `error` | **text** | An error description, if `status` is `failed. | -| `url` | **url** | The url which can now be used for downloading the application. | -| `expiresIn` | **int** | The time in seconds the url remains valid. | - -## category/list -Lists all the available categories. It uses the rank stored on the server for ordering. - -Returns a JSON array (not an object!). Each field is a JSON object: - -| JSON field | Description | -| ---------- | ----------- | -| `id` | The unique id of the category | -| `name` | The name of the category | - -## category/icon: -Returns an icon for the given category id. - -| Parameter | Description | -| ---------- | ----------- | -| `id` | The id of the category | - -Returns a PNG image or a 404 error diff --git a/appstore/settings.py b/appstore/settings.py index b03e943..bdd6e44 100644 --- a/appstore/settings.py +++ b/appstore/settings.py @@ -41,17 +41,16 @@ """ import os -APPSTORE_MAINTENANCE = False -APPSTORE_PLATFORM_ID = 'NEPTUNE3' -APPSTORE_PLATFORM_VERSION = 2 # Maximum supported platform version: - # version 1 - only old package format - # version 2 - old and new package formats -APPSTORE_DOWNLOAD_EXPIRY = 10 # in minutes -APPSTORE_BIND_TO_DEVICE_ID = True # unique downloads for each device -APPSTORE_NO_SECURITY = True # ignore developer signatures and do not generate store signatures -APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE = 'certificates/store.p12' -APPSTORE_STORE_SIGN_PKCS12_PASSWORD = 'password' -APPSTORE_DEV_VERIFY_CA_CERTIFICATES = ['certificates/ca.crt', 'certificates/devca.crt'] +APPSTORE_MAINTENANCE = False +APPSTORE_PLATFORM_ID = os.getenv('APPSTORE_PLATFORM_ID', default = 'NEPTUNE3') +APPSTORE_PLATFORM_VERSION = int(os.getenv('APPSTORE_PLATFORM_VERSION', default = '2')) +APPSTORE_DOWNLOAD_EXPIRY = int(os.getenv('APPSTORE_DOWNLOAD_EXPIRY', default = '10')) # in minutes +APPSTORE_BIND_TO_DEVICE_ID = os.getenv('APPSTORE_BIND_TO_DEVICE_ID', default = '1') == '1' # unique downloads for each device +APPSTORE_NO_SECURITY = os.getenv('APPSTORE_NO_SECURITY', default = '1') == '1' # ignore developer signatures and do not generate store signatures +APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE = os.getenv('APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE', default = 'certificates/store.p12') +APPSTORE_STORE_SIGN_PKCS12_PASSWORD = os.getenv('APPSTORE_STORE_SIGN_PKCS12_PASSWORD', default = 'password') +APPSTORE_DEV_VERIFY_CA_CERTIFICATES = os.getenv('APPSTORE_DEV_VERIFY_CA_CERTIFICATES', ','.join(['certificates/ca.crt', 'certificates/devca.crt'])).split(',') +APPSTORE_DATA_PATH = os.getenv('APPSTORE_DATA_PATH', default = '') # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(__file__)) @@ -61,12 +60,12 @@ # See https://docs.djangoproject.com/en/1.7/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = '4%(o_1zuz@^kjcarw&!5ptvk oa1-83*arn6jcm4idzy1#30' +SECRET_KEY = os.getenv('DJANGO_SECRET_KEY', default = '4%(o_1zuz@^kjcarw&!5ptvk oa1-83*arn6jcm4idzy1#30') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = True -ALLOWED_HOSTS = [] +ALLOWED_HOSTS = ['*'] TEMPLATES = [ { @@ -77,6 +76,7 @@ 'context_processors': ( "django.contrib.auth.context_processors.auth", "django.template.context_processors.debug", + "django.template.context_processors.request", "django.template.context_processors.i18n", "django.template.context_processors.media", "django.template.context_processors.static", @@ -102,12 +102,11 @@ 'store', ) -MIDDLEWARE_CLASSES = ( +MIDDLEWARE = ( 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', ) @@ -116,6 +115,12 @@ WSGI_APPLICATION = 'appstore.wsgi.application' +# Absolute path to the directory that holds media. +# Example: "/home/media/media.lawrence.com/" +if not APPSTORE_DATA_PATH: + MEDIA_ROOT = os.path.join(BASE_DIR, 'data/') +else: + MEDIA_ROOT = APPSTORE_DATA_PATH # Database # https://docs.djangoproject.com/en/1.7/ref/settings/#databases @@ -123,16 +128,16 @@ DATABASES = { 'default': { 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + 'NAME': os.path.join(MEDIA_ROOT, 'db.sqlite3'), } } # Internationalization # https://docs.djangoproject.com/en/1.7/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = os.getenv('DJANGO_LANGUAGE_CODE', default = 'en-us') -TIME_ZONE = 'Europe/Berlin' +TIME_ZONE = os.getenv('DJANGO_TIME_ZONE', default = 'Europe/Berlin') USE_I18N = True @@ -148,9 +153,6 @@ STATIC_URL = '/static/' -# Absolute path to the directory that holds media. -# Example: "/home/media/media.lawrence.com/" -MEDIA_ROOT = os.path.join(BASE_DIR, 'media/') STATIC_ROOT = os.path.join(BASE_DIR, 'static') # URL that handles the media served from MEDIA_ROOT. Make sure to use a @@ -164,3 +166,4 @@ # If the icon should be transformed to monochrome, with alpha channel, when uploaded or not ICON_DECOLOR = False +DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' diff --git a/appstore/urls.py b/appstore/urls.py index 1eb3d2c..667b899 100644 --- a/appstore/urls.py +++ b/appstore/urls.py @@ -30,26 +30,26 @@ ## ############################################################################# -from django.conf.urls import include, url +from django.urls import include, re_path from django.contrib import admin from store import api as store_api from appstore.settings import URL_PREFIX base_urlpatterns = [ - url(r'^admin/', include(admin.site.urls)), + re_path(r'^admin/', admin.site.urls), - url(r'^hello$', store_api.hello), - url(r'^login$', store_api.login), - url(r'^logout$', store_api.logout), - url(r'^app/list$', store_api.appList), - url(r'^app/icons/(.*)$', store_api.appIconNew), - url(r'^app/icon', store_api.appIcon), - url(r'^app/description', store_api.appDescription), - url(r'^app/purchase', store_api.appPurchase), - url(r'^app/download/(.*)$', store_api.appDownload), - url(r'^category/list$', store_api.categoryList), - url(r'^category/icon$', store_api.categoryIcon), - url(r'^upload$', store_api.upload), + re_path(r'^hello$', store_api.hello), + re_path(r'^login$', store_api.login), + re_path(r'^logout$', store_api.logout), + re_path(r'^app/list$', store_api.appList), + re_path(r'^app/icon$', store_api.appIcon), + re_path(r'^app/icons/(.*)$', store_api.appIconNew), + re_path(r'^app/description', store_api.appDescription), + re_path(r'^app/purchase', store_api.appPurchase), + re_path(r'^app/download/(.*)$', store_api.appDownload), + re_path(r'^category/list$', store_api.categoryList), + re_path(r'^category/icon$', store_api.categoryIcon), + re_path(r'^upload$', store_api.upload), ] @@ -58,6 +58,6 @@ prefix = prefix + URL_PREFIX + '/' urlpatterns = [ - url(prefix, include(base_urlpatterns)), + re_path(prefix, include(base_urlpatterns)), ] diff --git a/doc/QtAutoDeploymentServerDoc b/doc/QtAutoDeploymentServerDoc new file mode 100644 index 0000000..1c92d7c --- /dev/null +++ b/doc/QtAutoDeploymentServerDoc @@ -0,0 +1,3 @@ +// needed for the new clang based qdoc parser in Qt 5.11 +#include +#include diff --git a/doc/qtautodeploymentserver-project.qdocconf b/doc/qtautodeploymentserver-project.qdocconf index 258ef99..d310e7a 100644 --- a/doc/qtautodeploymentserver-project.qdocconf +++ b/doc/qtautodeploymentserver-project.qdocconf @@ -3,9 +3,14 @@ description = Qt Automotive Suite Deployment Server Documentat version = $QT_VERSION url = https://doc.qt.io/QtAutoDeploymentServer +# needed for the new clang based qdoc parser +moduleheader = QtAutoDeploymentServerDoc +includepaths = -I. + sourcedirs += src imagedirs += src/images +depends += qtcore qtapplicationmanager qhp.projects = QtAutoDeploymentServer qhp.QtAutoDeploymentServer.file = qtautodeploymentserver.qhp @@ -20,8 +25,8 @@ qhp.QtAutoDeploymentServer.customFilters.Qt.filterAttributes = qtautodeployments tagfile = qtautodeploymentserver.tags -depends = qtautomotivesuite - -buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" -navigation.homepage = "Qt Automotive Suite" navigation.landingpage = "Qt Automotive Suite Deployment Server" +buildversion = "Qt Automotive Suite Deployment Server $QT_VERSION" + +# Fail the documentation build if there are more warnings than the limit +warninglimit = 0 diff --git a/doc/src/deployment-server-http-server-setup.qdoc b/doc/src/deployment-server-http-server-setup.qdoc index b21ee76..d31b72c 100644 --- a/doc/src/deployment-server-http-server-setup.qdoc +++ b/doc/src/deployment-server-http-server-setup.qdoc @@ -27,13 +27,12 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-http-server-setup.html + \ingroup qtauto-deployment-server + \page http-server-setup.html \previouspage Qt Automotive Suite Deployment Server API Reference \nextpage Upload Packages to the Deployment Server - \contentspage {Qt Automotive Suite Deployment Server} \startpage Qt Automotive Suite Deployment Server - \title Set up a Production server with Apache, Lighttpd or Nginx The Deployment Server can be set up in combination with a regular web server: Apache, Lighttpd, or diff --git a/doc/src/deployment-server-installation.qdoc b/doc/src/deployment-server-installation.qdoc index 7937992..06fb895 100644 --- a/doc/src/deployment-server-installation.qdoc +++ b/doc/src/deployment-server-installation.qdoc @@ -27,14 +27,24 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-installation.html - \contentspage {Qt Automotive Suite Deployment Server} + \ingroup qtauto-deployment-server + \page installation.html \previouspage Qt Automotive Suite Deployment Server \nextpage Qt Automotive Suite Deployment Server API Reference \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server Installation + \section1 Set up the Server in a Docker Container + + The new recommended way to run this server is through the supplied \c Dockerfile and the + \c docker-manage.sh script, which can both be found in the modules root directory. + + Instead of messing with Django's project configuration, you can simply export your individual + settings as environment variables. Either directly in the \c Dockerfile when building the + container, or by copying and modifying the \c docker-manage.sh script. + + \section1 Set up the Server in a Virtual Environment Before you install the dependencies in the Python virtual environment, you need to install the diff --git a/doc/src/deployment-server-package-upload.qdoc b/doc/src/deployment-server-package-upload.qdoc index dc5b7b4..f2e09de 100644 --- a/doc/src/deployment-server-package-upload.qdoc +++ b/doc/src/deployment-server-package-upload.qdoc @@ -27,10 +27,10 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-package-upload.html + \ingroup qtauto-deployment-server + \page package-upload.html \previouspage Set up a Production server with Apache, Lighttpd or Nginx \nextpage Qt Automotive Suite Deployment Server - \contentspage {Qt Automotive Suite Deployment Server} \startpage Qt Automotive Suite Deployment Server @@ -43,7 +43,7 @@ server itself. \section1 Through Server Admin Page This was the first uploading method implemented. It uses django admin page, accessible by \c{/admin/} URL of -the Deployment Server. For Qt 5.14, the URL is \l{http://demoappsdeploy.qt.io:8514/admin/}. +the Deployment Server. To add application: \list diff --git a/doc/src/deployment-server-reference.qdoc b/doc/src/deployment-server-reference.qdoc index d4be3af..6036e67 100644 --- a/doc/src/deployment-server-reference.qdoc +++ b/doc/src/deployment-server-reference.qdoc @@ -27,10 +27,10 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-reference.html + \ingroup qtauto-deployment-server + \page reference.html \previouspage Qt Automotive Suite Deployment Server Installation \nextpage Set up a Production server with Apache, Lighttpd or Nginx - \contentspage {Qt Automotive Suite Deployment Server} \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server API Reference diff --git a/doc/src/deployment-server.qdoc b/doc/src/deployment-server.qdoc index 50f2bee..9c6390c 100644 --- a/doc/src/deployment-server.qdoc +++ b/doc/src/deployment-server.qdoc @@ -27,13 +27,19 @@ ****************************************************************************/ /*! - \page qtauto-deployment-server-index.html - \contentspage {Qt Automotive Suite} + \ingroup qtauto-deployment-server + \page index.html \nextpage Qt Automotive Suite Deployment Server Installation \startpage Qt Automotive Suite Deployment Server \title Qt Automotive Suite Deployment Server + \note This project is not maintained anymore. It was ported to a Qt 6 cmake setup and a more + modern Django and Python version to at least keep it usable for legacy projects. + For non-production use-cases, please switch to the new + \l{Package-Server}{appman-package-server} + available in the \l{Qt Application Manager} starting with version 6.7. + The Qt Automotive Suite Deployment Server is a new component in the Qt Automotive Suite 5.12. Previously, it was known as the Neptune Appstore and used for demonstrations purposes. diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh new file mode 100755 index 0000000..6e7fde5 --- /dev/null +++ b/docker-entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd /server +exec ./manage.py "$@" diff --git a/docker-manage.sh b/docker-manage.sh new file mode 100755 index 0000000..e1cef29 --- /dev/null +++ b/docker-manage.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +#export APPSTORE_PLATFORM_ID NEPTUNE3 +#export APPSTORE_PLATFORM_VERSION 2 +#export APPSTORE_DOWNLOAD_EXPIRY 10 +#export APPSTORE_BIND_TO_DEVICE_ID 1 +#export APPSTORE_NO_SECURITY 1 +#export APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE certificates/store.p12 +#export APPSTORE_STORE_SIGN_PKCS12_PASSWORD password +#export APPSTORE_DEV_VERIFY_CA_CERTIFICATES certificates/ca.crt,certificates/devca.crt + +IT="" +if [ "x$1" = "x-it" ]; then + shift + IT=-it +fi + +cd `dirname $0`/.. +mkdir -p data + +exec docker run $IT \ + -p 8080:8080 \ + -v `pwd`/data:/data \ + -e APPSTORE_PLATFORM_ID \ + -e APPSTORE_PLATFORM_VERSION \ + -e APPSTORE_DOWNLOAD_EXPIRY \ + -e APPSTORE_BIND_TO_DEVICE_ID \ + -e APPSTORE_NO_SECURITY \ + -e APPSTORE_STORE_SIGN_PKCS12_CERTIFICATE \ + -e APPSTORE_STORE_SIGN_PKCS12_PASSWORD \ + -e APPSTORE_DEV_VERIFY_CA_CERTIFICATES \ + qtauto-deployment-server "$@" diff --git a/manage.py b/manage.py index af588f4..ea5aa0a 100755 --- a/manage.py +++ b/manage.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/env python3 ############################################################################# ## ## Copyright (C) 2019 Luxoft Sweden AB diff --git a/qtauto-deployment-server.pro b/qtauto-deployment-server.pro deleted file mode 100644 index f7f9002..0000000 --- a/qtauto-deployment-server.pro +++ /dev/null @@ -1,13 +0,0 @@ -TEMPLATE = aux - -build_online_docs: { - QMAKE_DOCS = $$PWD/doc/online/qtautodeploymentserver.qdocconf -} else { - QMAKE_DOCS = $$PWD/doc/qtautodeploymentserver.qdocconf -} - -OTHER_FILES += \ - $$PWD/doc/*.qdocconf \ - $$PWD/doc/src/*.qdoc - -load(qt_docs) diff --git a/requirements.txt b/requirements.txt index 92c71f9..4693615 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYAML -django==1.11.27 +django==4.0.6 django-common -django-ordered-model==2.1 +django-ordered-model pyOpenSSL M2Crypto Enum34 @@ -10,4 +10,4 @@ cffi paramiko cryptography pillow -python-magic==0.4.15 +python-magic diff --git a/store/admin.py b/store/admin.py index 910f3fb..389bb87 100644 --- a/store/admin.py +++ b/store/admin.py @@ -30,15 +30,14 @@ ## ############################################################################# -import StringIO +import io from PIL import Image, ImageChops from django import forms from django.contrib import admin -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy from django.core.files.uploadedfile import InMemoryUploadedFile from ordered_model.admin import OrderedModelAdmin +from django.utils.safestring import mark_safe from store.models import * from store.utilities import parseAndValidatePackageMetadata, writeTempIcon, makeTagList @@ -68,10 +67,11 @@ def clean(self): im = Image.open(cleaned_data['icon']) size = (settings.ICON_SIZE_X, settings.ICON_SIZE_Y) im.thumbnail(size, Image.ANTIALIAS) - imagefile = StringIO.StringIO() + imagefile = io.BytesIO() im.save(imagefile, format='png') + length = imagefile.tell() imagefile.seek(0) - cleaned_data['icon'] = InMemoryUploadedFile(imagefile, 'icon', "icon.png", 'image/png', imagefile.len, None) + cleaned_data['icon'] = InMemoryUploadedFile(imagefile, 'icon', "icon.png", 'image/png', length, None) return cleaned_data class CategoryAdmin(OrderedModelAdmin): @@ -85,16 +85,15 @@ def save_model(self, request, obj, form, change): def name(self, obj): # just to forbid sorting by name return obj.name - name.short_description = ugettext_lazy('Item caption') + name.short_description = u'Item caption' def icon_image(self, obj): prefix = settings.URL_PREFIX image_request = prefix + "/category/icon?id=%s" % (obj.id) - html = u'' % (settings.ICON_SIZE_X, settings.ICON_SIZE_Y, image_request) - return html + html = '' % (settings.ICON_SIZE_X, settings.ICON_SIZE_Y, image_request) + return mark_safe(html) - icon_image.allow_tags = True - icon_image.short_description = ugettext_lazy('Category icon') + icon_image.short_description = u'Category icon' class AppAdminForm(forms.ModelForm): @@ -113,7 +112,7 @@ def clean(self): try: pkgdata = parseAndValidatePackageMetadata(package_file) except Exception as error: - raise forms.ValidationError(_('Validation error: %s' % str(error))) + raise forms.ValidationError('Validation error: %s' % str(error)) self.appId = pkgdata['info']['id'] self.name = pkgdata['storeName'] @@ -123,27 +122,27 @@ def clean(self): # check if this really is an update if hasattr(self, 'instance') and self.instance.appid: if self.appId != self.instance.appid: - raise forms.ValidationError(_('Validation error: an update cannot change the ' - 'application id, tried to change from %s to %s' % - (self.instance.appid, self.appId))) + raise forms.ValidationError('Validation error: an update cannot change the ' + 'application id, tried to change from %s to %s' % + (self.instance.appid, self.appId)) elif self.architecture != self.instance.architecture: - raise forms.ValidationError(_('Validation error: an update cannot change the ' - 'application architecture from %s to %s' % - (self.instance.architecture, self.architecture))) + raise forms.ValidationError('Validation error: an update cannot change the ' + 'application architecture from %s to %s' % + (self.instance.architecture, self.architecture)) else: try: if App.objects.get(appid__exact=self.appId, architecture__exact=self.architecture, tags_hash__exact=self.tags_hash): - raise forms.ValidationError(_('Validation error: another application with id' - ' %s , tags %s and architecture %s already ' - 'exists' % (str(self.appId), str(self.tags_hash), - str(self.architecture)))) + raise forms.ValidationError('Validation error: another application with id' + ' %s , tags %s and architecture %s already ' + 'exists' % (str(self.appId), str(self.tags_hash), + str(self.architecture))) except App.DoesNotExist: pass # write icon into file to serve statically success, error = writeTempIcon(self.appId, self.architecture, self.tags_hash, pkgdata['icon']) if not success: - raise forms.ValidationError(_(error)) + raise forms.ValidationError(error) return cleaned_data diff --git a/store/api.py b/store/api.py index 9444b0b..b617e34 100644 --- a/store/api.py +++ b/store/api.py @@ -33,6 +33,7 @@ import os import shutil import hashlib +import logging from django.conf import settings from django.db.models import Q, Count @@ -89,7 +90,6 @@ def hello(request): else: request.session['architecture'] = '' - request.session['pkgversions'] = range(1, version + 1) return JsonResponse({'status': status}) @@ -120,7 +120,7 @@ def login(request): def logout(request): status = 'ok' - if not request.user.is_authenticated(): + if not request.user.is_authenticated: status = 'failed' logout(request) @@ -201,12 +201,7 @@ def appList(request): if 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] - apps = apps.filter(architecture__in=archlist) - apps = apps.filter(pkgformat__in=versionlist) #Tag filtering #There is no search by version distance yet - this must be fixed @@ -261,13 +256,9 @@ def appDescription(request): archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] appId = getRequestDictionary(request)['id'] try: app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') - app = app.filter(pkgformat__in=versionlist) #Tag filtering #There is no search by version distance yet - this must be fixed if 'tag' in request.session: @@ -285,7 +276,7 @@ def appIconNew(request, path): path=path.replace('/', '_').replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png' try: response = HttpResponse(content_type='image/png') - with open(iconPath() + path, 'rb') as pkg: + with open(os.path.join(settings.MEDIA_ROOT, iconPath(), path), 'rb') as pkg: response.write(pkg.read()) response['Content-Length'] = pkg.tell() return response @@ -299,22 +290,20 @@ def appIcon(request): archlist.append(normalizeArch(dictionary['architecture'])) elif 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] appId = dictionary['id'] try: - app = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') - app = app.filter(pkgformat__in=versionlist) + apps = App.objects.filter(appid__exact = appId, architecture__in = archlist).order_by('architecture','tags_hash') #Tag filtering #There is no search by version distance yet - this must be fixed if 'tag' in request.session: tags = SoftwareTagList() tags.parse(request.session['tag']) - app_ids = [x.id for x in app if x.is_tagmatching(tags.list())] - app = App.objects.filter(id__in=app_ids) - app = app.last() - with open(iconPath(app.appid, app.architecture, app.tags_hash), 'rb') as iconPng: + app_ids = [x.id for x in apps if x.is_tagmatching(tags.list())] + apps = App.objects.filter(id__in=app_ids) + app = apps.last() + path = iconPath(app.appid, app.architecture, app.tags_hash) + path = os.path.join(settings.MEDIA_ROOT, path) + with open(path, 'rb') as iconPng: response = HttpResponse(content_type='image/png') response.write(iconPng.read()) return response @@ -323,14 +312,11 @@ def appIcon(request): def appPurchase(request): - if not request.user.is_authenticated(): + if not request.user.is_authenticated: return HttpResponseForbidden('no login') archlist = ['All', ] if 'architecture' in request.session: archlist.append(request.session['architecture']) - versionlist = [1] - if 'pkgversions' in request.session: - versionlist = request.session['pkgversions'] try: deviceId = str(getRequestDictionary(request).get("device_id", "")) @@ -346,7 +332,6 @@ def appPurchase(request): app = App.objects.filter(id__exact = getRequestDictionary(request)['purchaseId'], architecture__in=archlist).order_by('architecture','tags_hash') else: raise ValidationError('id or purchaseId parameter required') - app = app.filter(pkgformat__in=versionlist) #Tag filtering #There is no search by version distance yet - this must be fixed if 'tag' in request.session: @@ -356,7 +341,7 @@ def appPurchase(request): app = App.objects.filter(id__in=app_ids) app = app.last() - fromFilePath = packagePath(app.appid, app.architecture, app.tags_hash) + fromFilePath = os.path.join(settings.MEDIA_ROOT, packagePath(app.appid, app.architecture, app.tags_hash)) # we should not use obvious names here, but just hash the string. # this would be a nightmare to debug though and this is a development server :) @@ -423,7 +408,7 @@ def categoryIcon(request): try: if categoryId != '-1': category = Category.objects.filter(id__exact = categoryId)[0] - filename = iconPath() + "category_" + str(category.id) + ".png" + filename = os.path.join(settings.MEDIA_ROOT, iconPath(), "category_" + str(category.id) + ".png") else: from django.contrib.staticfiles import finders filename = finders.find('img/category_All.png') @@ -439,7 +424,7 @@ def categoryIcon(request): # |Error| # | | # +-----+ - emptyPng = "\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x01\x03\x00\x00\x00I\xb4\xe8\xb7\x00\x00\x00\x06PLTE\x00\x00\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\x33IDAT\x08\xd7\x63\xf8\x0f\x04\x0c\x0d\x0c\x0c\x8c\x44\x13\x7f\x40\xc4\x01\x10\x71\xb0\xf4\x5c\x2c\xc3\xcf\x36\xc1\x44\x86\x83\x2c\x82\x8e\x48\xc4\x5f\x16\x3e\x47\xd2\x0c\xc5\x46\x80\x9c\x06\x00\xa4\xe5\x1d\xb4\x8e\xae\xe8\x43\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82" + emptyPng = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00 \x00\x00\x00 \x01\x03\x00\x00\x00I\xb4\xe8\xb7\x00\x00\x00\x06PLTE\x00\x00\x00\x00\x00\x00\xa5\x67\xb9\xcf\x00\x00\x00\x01tRNS\x00@\xe6\xd8f\x00\x00\x00\x33IDAT\x08\xd7\x63\xf8\x0f\x04\x0c\x0d\x0c\x0c\x8c\x44\x13\x7f\x40\xc4\x01\x10\x71\xb0\xf4\x5c\x2c\xc3\xcf\x36\xc1\x44\x86\x83\x2c\x82\x8e\x48\xc4\x5f\x16\x3e\x47\xd2\x0c\xc5\x46\x80\x9c\x06\x00\xa4\xe5\x1d\xb4\x8e\xae\xe8\x43\x00\x00\x00\x00\x49\x45\x4e\x44\xae\x42\x60\x82" response.write(emptyPng) return response diff --git a/store/authdecorators.py b/store/authdecorators.py index 2a4119c..a4307ff 100644 --- a/store/authdecorators.py +++ b/store/authdecorators.py @@ -60,7 +60,7 @@ def view_or_basicauth(view, request, test_func, realm="", *args, **kwargs): # NOTE: We are only support basic authentication for now. # if auth[0].lower() == "basic": - uname, passwd = base64.b64decode(auth[1]).split(':') + uname, passwd = base64.b64decode(auth[1].encode('utf-8')).decode('utf-8').split(':') user = authenticate(username=uname, password=passwd) if user is not None: if user.is_active: @@ -114,7 +114,7 @@ def your_view: def view_decorator(func): def wrapper(request, *args, **kwargs): return view_or_basicauth(func, request, - lambda u: u.is_authenticated(), + lambda u: u.is_authenticated, realm, *args, **kwargs) return wrapper diff --git a/store/management/commands/expire-downloads.py b/store/management/commands/expire-downloads.py index 94b0d24..012182e 100644 --- a/store/management/commands/expire-downloads.py +++ b/store/management/commands/expire-downloads.py @@ -39,7 +39,7 @@ from store.utilities import downloadPath class Command(BaseCommand): - help = 'Expires all downloads that are older than 10 minutes' + help = 'Expires all downloads that are older than APPSTORE_DOWNLOAD_EXPIRY minutes' def handle(self, *args, **options): self.stdout.write('Removing expired download packages') @@ -50,7 +50,7 @@ def handle(self, *args, **options): for pkg in os.listdir(pkgPath): t = os.path.getmtime(pkgPath + pkg) age = time.time() - t - if age > (10 * 60): + if age > (int(settings.APPSTORE_DOWNLOAD_EXPIRY) * 60): os.remove(pkgPath + pkg) self.stdout.write(' -> %s (age: %s seconds)' % (pkg, int(age))) diff --git a/store/migrations/0001_initial.py b/store/migrations/0001_initial.py index 80d154c..2bb3457 100644 --- a/store/migrations/0001_initial.py +++ b/store/migrations/0001_initial.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -# Generated by Django 1.11.27 on 2020-08-14 17:24 ############################################################################# ## ## Copyright (C) 2020 Luxoft Sweden AB @@ -32,7 +31,7 @@ ## ############################################################################# -# Generated by Django 1.11.27 on 2020-07-22 16:39 +# Generated by Django 1.11.27 on 2020-10-30 13:46 from __future__ import unicode_literals from django.conf import settings @@ -62,9 +61,9 @@ class Migration(migrations.Migration): ('description', models.TextField()), ('dateAdded', models.DateField(auto_now_add=True)), ('dateModified', models.DateField(auto_now=True)), - ('tags_hash', models.CharField(default=b'', max_length=4096)), - ('architecture', models.CharField(default=b'All', max_length=20)), - ('version', models.CharField(default=b'0.0.0', max_length=20)), + ('tags_hash', models.CharField(default='', max_length=4096)), + ('architecture', models.CharField(default='All', max_length=20)), + ('version', models.CharField(default='0.0.0', max_length=20)), ('pkgformat', models.IntegerField()), ], ), diff --git a/store/migrations/0002_alter_category_order.py b/store/migrations/0002_alter_category_order.py new file mode 100644 index 0000000..e13175c --- /dev/null +++ b/store/migrations/0002_alter_category_order.py @@ -0,0 +1,18 @@ +# Generated by Django 4.0.6 on 2023-06-26 19:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('store', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='category', + name='order', + field=models.PositiveIntegerField(db_index=True, editable=False, verbose_name='order'), + ), + ] diff --git a/store/models.py b/store/models.py index 7e8751a..ff8ab0c 100644 --- a/store/models.py +++ b/store/models.py @@ -45,7 +45,7 @@ def category_file_name(instance, filename): # filename parameter is unused. See django documentation for details: # https://docs.djangoproject.com/en/1.11/ref/models/fields/#django.db.models.FileField.upload_to - return settings.MEDIA_ROOT + "icons/category_" + str(instance.id) + ".png" + return "icons/category_" + str(instance.id) + ".png" class OverwriteStorage(FileSystemStorage): def get_available_name(self, name, max_length=None): @@ -63,6 +63,9 @@ class Meta(OrderedModel.Meta): def __unicode__(self): return self.name + def __str__(self): + return self.name + def save(self, *args, **kwargs): if self.id is None: # This is a django hack. When category icon is saved and then later accessed, @@ -77,13 +80,16 @@ def save(self, *args, **kwargs): super(Category, self).save(*args, **kwargs) class Vendor(models.Model): - user = models.ForeignKey(User, primary_key = False) + user = models.ForeignKey(User, primary_key = False, on_delete = models.CASCADE) name = models.CharField(max_length = 200) certificate = models.TextField(max_length = 8000) def __unicode__(self): return self.name + def __str__(self): + return self.name + class Tag(models.Model): negative = models.BooleanField(default=False) name = models.CharField(max_length=200) @@ -106,8 +112,8 @@ class App(models.Model): appid = models.CharField(max_length=200) name = models.CharField(max_length=200) file = models.FileField(upload_to=content_file_name, storage=OverwriteStorage()) - vendor = models.ForeignKey(Vendor) - category = models.ForeignKey(Category) + vendor = models.ForeignKey(Vendor, on_delete = models.CASCADE) + category = models.ForeignKey(Category, on_delete = models.CASCADE) briefDescription = models.TextField() description = models.TextField() dateAdded = models.DateField(auto_now_add=True) diff --git a/store/tags.py b/store/tags.py index e906684..da09803 100644 --- a/store/tags.py +++ b/store/tags.py @@ -79,7 +79,7 @@ def __init__(self, tag): """ Takes tag and parses it. If it can't parse - raises exception of invalid value :type tag: str """ - if not isinstance(tag, (str, unicode)): + if not isinstance(tag, str): raise BaseException("Invalid input data-type") if not validateTag(tag): raise BaseException("Malformed tag") @@ -121,7 +121,7 @@ def __init__(self): def __str__(self): lst = list() - for _, value in self.taglist.items(): + for _, value in list(self.taglist.items()): lst += [str(i) for i in value] lst.sort() return ",".join(lst) @@ -168,7 +168,7 @@ def is_empty(self): def list(self): lst = list() - for _, value in self.taglist.items(): + for _, value in list(self.taglist.items()): for i in value: lst.append(i) return lst @@ -188,13 +188,13 @@ def test_tag_creation(self): self.assertTrue(tag.has_version()) tag = SoftwareTag('Qt') self.assertFalse(tag.has_version()) - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): SoftwareTag('фыва') - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): SoftwareTag('фыва:5.1') - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): SoftwareTag('qt.1:5.1') - with self.assertRaisesRegexp(BaseException, "Invalid input data-type"): + with self.assertRaisesRegex(BaseException, "Invalid input data-type"): SoftwareTag(1) def test_tag_match(self): @@ -227,11 +227,11 @@ def test_not_empty_after_append(self): def test_append_invalid(self): lst = SoftwareTagList() - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): self.assertFalse(lst.append(SoftwareTag('qt:1:1'))) # Invalid version - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): self.assertFalse(lst.append(SoftwareTag('фыва'))) # Non-ascii - with self.assertRaisesRegexp(BaseException, "Malformed tag"): + with self.assertRaisesRegex(BaseException, "Malformed tag"): self.assertFalse(lst.append(SoftwareTag(''))) # empty tag is not valid def test_append_valid(self): diff --git a/store/utilities.py b/store/utilities.py index 06151bb..1a283e5 100644 --- a/store/utilities.py +++ b/store/utilities.py @@ -70,28 +70,29 @@ def getRequestDictionary(request): return request.GET def packagePath(appId=None, architecture=None, tags=None): - path = settings.MEDIA_ROOT + 'packages/' + path = "packages" ## os.path.join(settings.MEDIA_ROOT, 'packages/') if tags is None: tags = "" if (appId is not None) and (architecture is not None): - path = path + '_'.join([appId, architecture, tags]).replace('/', '_').\ - replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + path = os.path.join(path, '_'.join([appId, architecture, tags]).replace('/', '_').\ + replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C')) return path def iconPath(appId=None, architecture=None, tags=None): - path = settings.MEDIA_ROOT + 'icons/' + path = "icons" ## os.path.join(settings.MEDIA_ROOT, 'icons/') if tags is None: tags = "" if (appId is not None) and (architecture is not None): - return path + '_'.join([appId, architecture, tags]).replace('/', '_').\ - replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png' + return os.path.join(path, '_'.join([appId, architecture, tags]).replace('/', '_').\ + replace('\\', '_').replace(':', 'x3A').replace(',', 'x2C') + '.png') return path def writeTempIcon(appId, architecture, tags, icon): try: - if not os.path.exists(iconPath()): - os.makedirs(iconPath()) - tempicon = open(iconPath(appId, architecture, tags), 'w') + path = os.path.join(settings.MEDIA_ROOT, iconPath()) + if not os.path.exists(path): + os.makedirs(path) + tempicon = open(os.path.join(settings.MEDIA_ROOT, iconPath(appId, architecture, tags)), 'wb') tempicon.write(icon) tempicon.flush() tempicon.close() @@ -101,39 +102,35 @@ def writeTempIcon(appId, architecture, tags, icon): str(error) def downloadPath(): - return settings.MEDIA_ROOT + 'downloads/' + return os.path.join(settings.MEDIA_ROOT, 'downloads/') -def isValidDnsName(dnsName, errorList): - # see also in AM: src/common-lib/utilities.cpp / isValidDnsName() +def isValidFilesystemName(name, errorList): + # see also in AM: src/common-lib/utilities.cpp / validateForFilesystemUsage try: - # this is not based on any RFC, but we want to make sure that this id is usable as filesystem - # name. So in order to support FAT (SD-Cards), we need to keep the path < 256 characters + # we need to make sure that we can use the name as directory in a filesystem and inode names + # are limited to 255 characters in Linux. We need to subtract a safety margin for prefixes + # or suffixes though: - if len(dnsName) > 200: - raise Exception('too long - the maximum length is 200 characters') + if not name: + raise Exception('must not be empty') - # we require at least 3 parts: tld.company-name.application-name - # this make it easier for humans to identify apps by id. + if len(name) > 150: + raise Exception('the maximum length is 150 characters') - labels = dnsName.split('.') - if len(labels) < 3: - raise Exception('wrong format - needs to be in reverse-DNS notation and consist of at least three parts separated by .') + # all characters need to be ASCII minus any filesystem special characters: + spaceOnly = True + forbiddenChars = '<>:"/\\|?*' + for i, c in enumerate(name): + if (ord(c) < 0x20) or (ord(c) > 0x7f) or (c in forbiddenChars): + raise Exception(f'must consist of printable ASCII characters only, except any of \'{forbiddenChars}\'') - # standard domain name requirements from the RFCs 1035 and 1123 + if spaceOnly: + spaceOnly = (c == ' ') - for label in labels: - if 0 >= len(label) > 63: - raise Exception('wrong format - each part of the name needs to at least 1 and at most 63 characters') - - for i, c in enumerate(label): - isAlpha = (c >= '0' and c <= '9') or (c >= 'a' and c <= 'z'); - isDash = (c == '-'); - isInMiddle = (i > 0) and (i < (len(label) - 1)); - - if not (isAlpha or (isDash and isInMiddle)): - raise Exception('invalid characters - only [a-z0-9-] are allowed (and '-' cannot be the first or last character)') + if spaceOnly: + raise Exception('must not consist of only white-space characters') return True @@ -141,7 +138,6 @@ def isValidDnsName(dnsName, errorList): errorList[0] = str(error) return False - def verifySignature(signaturePkcs7, hash, chainOfTrust): # see also in AM: src/crypto-lib/signature.cpp / Signature::verify() @@ -256,7 +252,7 @@ def parsePackageMetadata(packageFile): raise Exception('the first file in the package is not --PACKAGE-HEADER--, but %s' % entry.name) if entry.name.startswith('--PACKAGE-FOOTER--'): - footerContents += contents + footerContents += contents.decode('utf-8') foundFooter = True elif foundFooter: @@ -264,10 +260,11 @@ def parsePackageMetadata(packageFile): if not entry.name.startswith('--PACKAGE-'): addToDigest1 = '%s/%s/' % ('D' if entry.isdir() else 'F', 0 if entry.isdir() else entry.size) + addToDigest1 = addToDigest1.encode('utf-8') entryName = entry.name if entry.isdir() and entryName.endswith('/'): entryName = entryName[:-1] - addToDigest2 = unicode(entryName, 'utf-8').encode('utf-8') + addToDigest2 = str(entryName).encode('utf-8') if entry.isfile(): digest.update(contents) @@ -380,7 +377,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): 'icon': [], 'digest': [] } - for part in partFields.keys(): + for part in list(partFields.keys()): if not part in pkgdata: raise Exception('package metadata is missing the %s part' % part) data = pkgdata[part] @@ -393,7 +390,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): raise Exception('the id fields in --PACKAGE-HEADER-- and info.yaml are different: %s vs. %s' % (pkgdata['header'][packageIdKey], pkgdata['info']['id'])) error = [''] - if not isValidDnsName(pkgdata['info']['id'], error): + if not isValidFilesystemName(pkgdata['info']['id'], error): raise Exception('invalid id: %s' % error[0]) if pkgdata['header']['diskSpaceUsed'] <= 0: @@ -408,7 +405,7 @@ def parseAndValidatePackageMetadata(packageFile, certificates = []): elif 'en_US' in pkgdata['info']['name']: name = pkgdata['info']['name']['en_US'] elif len(pkgdata['info']['name']) > 0: - name = pkgdata['info']['name'].values()[0] + name = list(pkgdata['info']['name'].values())[0] if not name: raise Exception('could not deduce a suitable package name from the info part') @@ -456,7 +453,7 @@ def addFileToPackage(sourcePackageFile, destinationPackageFile, fileName, fileCo entry = dst.gettarinfo(fileobj = tmp, arcname = fileName) entry.uid = entry.gid = 0 entry.uname = entry.gname = '' - entry.mode = 0400 + entry.mode = 0o400 dst.addfile(entry, fileobj = tmp) dst.close() diff --git a/tests/tests.pro b/tests/tests.pro deleted file mode 100644 index 9671085..0000000 --- a/tests/tests.pro +++ /dev/null @@ -1 +0,0 @@ -TEMPLATE = subdirs