diff --git a/.eslintrc.js b/.eslintrc.js new file mode 100644 index 00000000..4a5b9ce0 --- /dev/null +++ b/.eslintrc.js @@ -0,0 +1,29 @@ +module.exports = { + "env": { + "node": true, + "mocha": true + }, + "extends": "standard", + "parserOptions": { + "ecmaVersion": 6 + }, + "rules": { + "indent": [ + "error", + 4 + ], + "linebreak-style": [ + "error", + "unix" + ], + "quotes": [ + "error", + "single" + ], + "semi": [ + "error", + "always" + ], + "eqeqeq": [0] + } +}; diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml new file mode 100644 index 00000000..a4d3087e --- /dev/null +++ b/.github/workflows/ci-test.yml @@ -0,0 +1,34 @@ +name: NodeJS CI with NPM + +on: [push] + +jobs: + build: + strategy: + fail-fast: false + max-parallel: 1 + matrix: + node_version: ['6', '8', '10', '12', '14', '16'] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Setup NodeJS + uses: actions/setup-node@v2 + with: + node-version: ${{ matrix.node_version }} + - name: Setup dependencies + run: | + npm install + npm install -g codecov@3.8.1 + npm install -g istanbul + - name: Run cases + run: | + npm test + istanbul cover ./node_modules/mocha/bin/_mocha --reporter test -- -R spec + codecov + env: + QINIU_ACCESS_KEY: ${{ secrets.QINIU_ACCESS_KEY }} + QINIU_SECRET_KEY: ${{ secrets.QINIU_SECRET_KEY }} + QINIU_TEST_BUCKET: ${{ secrets.QINIU_TEST_BUCKET }} + QINIU_TEST_DOMAIN: ${{ secrets.QINIU_TEST_DOMAIN }} diff --git a/.gitignore b/.gitignore index 369eea93..022d82dc 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ lib-cov *.out *.pid *.gz +test-env.sh pids logs @@ -18,3 +19,9 @@ results node_modules npm-debug.log test/config.js +package-lock.json + +coverage/ +.nyc_output/ +.idea +yarn.lock diff --git a/.npmignore b/.npmignore index 3867aec4..ab8c923e 100644 --- a/.npmignore +++ b/.npmignore @@ -1,7 +1,12 @@ test/ test-env.sh .travis.yml +.eslintrc.js coverage.html +gopher.png lib-cov/ Makefile docs/ +examples/ +coverage/ +.nyc_output/ diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 9aec1f50..00000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -language: node_js -node_js: -- '4' -- '6' -- '8' -before_script: -- export is_travis=true -deploy: - provider: npm - email: sdk@qiniu.com - api_key: - secure: GbKSsROWx6J33WK23cT08eizlkRqpbnCEv3bh4I9R8K5HuUzk08TkNj/LxxuWOEG3hrthivnb5gxshE5UxJ8+nGkS1N2lgZX1i8qT6dGrHb3xda7QB1Szm5+305fCpIa/0jsFqud/8l0GNsNyLezeoaihaNAb9Et5tz6JYVfQxU= - on: - tags: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 4da67e8b..055d11eb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,101 @@ ## CHANGE LOG +## 7.11.1 +- 对象存储，上传策略移除严格模式，支持任意策略选项 -## 7.1.3 +## 7.11.0 +- 对象存储，新增支持归档直读存储 +- 对象存储，批量操作、解冻操作支持自动查询 rs 服务域名 + +## 7.10.1 +- 对象存储，修复无法上传带有英文双引号的文件 + +## 7.10.0 +- 对象存储，上传支持双活 +- 对象存储，上传回调支持 Promise 风格 +- 对象存储，修复分片上传v2在创建 uploadId 失败后仍尝试上传 + +## 7.9.0 +- 对象存储，修复无法对 key 为空字符串的对象进行操作 +- 对象存储，查询区域域名支持配置 UC 地址 +- 对象存储，查询区域域名接口升级 +- 对象存储，更新设置镜像源的域名 +- 对象存储，新增请求中间件逻辑，方便拓展请求逻辑 +- 对象存储，新增备用 UC 域名用于查询区域域名 +- 对象存储，移除首尔区域 +- 对象存储，新增 华东浙江2 区域的类型声明 + +## 7.8.0 +- 移除不推荐域名，并增加 亚太-首尔 和 华东-浙江2 固定区域 +- RTC，优化请求失败的错误信息 +- 对象存储，优化分片上传 ctx 超时检测 +- 对象存储，修复事件、跨域部分 API 身份认证错误（v7.7.0 引入） +- 对象存储，新增事件、跨域 API 的类型声明（.d.ts） + +## 7.7.0 +- 对象存储，管理类 API 发送请求时增加 [X-Qiniu-Date](https://developer.qiniu.com/kodo/3924/common-request-headers) （生成请求的时间） header + +## 7.6.1 +- 添加sms typing + +## 7.6.0 +- 对象存储，新增 setObjectLifeCycle 设置对象生命周期 API + +## 7.5.0 +- 对象存储，新增支持 [深度归档存储类型](https://developer.qiniu.com/kodo/3956/kodo-category#deep_archive) +- 对象存储，修复在不制定区域信息情况下自动获取区域信息异常问题 +- 修复 Qiniu 签名算法在对部分 http header 处理异常问题 + +## 7.4.0 +- 支持 [分片上传 V2](https://developer.qiniu.com/kodo/6364/multipartupload-interface) + + +## 7.3.3 +- 修复上传策略中forceSaveKey指定无效 + +## 7.3.2 +- 修复crc32指定值无效 +- 修复checkCrc指定无效 +- 统一checkCrc类型 + +## 7.3.1 +- 新增归档存储解冻接口 +- 支持上传时key值为空字符串 + +## v7.3.0 +- 新增 19 个bucket及文件相关操作 +- 新增文件列举 v2 接口功能 +- 新增短信功能 +- 更新上传加速域名策略 +- 修复自动获取上传区域时的无效缓存 +- 修复大文件上传异常 +- 增加测试用例 + +## v7.2.2 +- 一些log输出问题，travis增加eslint 检查 + +## v7.2.1 +- 修复rtc获取回复存在的问题 + +## v7.2.0 +- 修复node的stream读取的chunk大小比较随意的问题 + +## v7.1.9 +- 修复新版node下resume up方式文件内容被缓存而导致的上传失败 + +## v7.1.8 +- 修复 index.d.ts 文件中zone的设置 + +## v7.1.7 +- 修复form上传在升级mime库后的错误 + +## v7.1.6 +- 修复rs和rsf的https默认域名 +- 升级修复mime库的安全风险 + +## v7.1.5 +- 增加连麦功能 + +## v7.1.3 - 增加新加坡机房 ## v7.1.2 diff --git a/Makefile b/Makefile index 5e707ad3..854e48b0 100644 --- a/Makefile +++ b/Makefile @@ -1,22 +1,11 @@ -TESTS = test/*.test.js -TIMEOUT = 25000 -REPORTER = spec -MOCHA_OPTS = test: - @NODE_ENV=test ./node_modules/.bin/mocha \ - --require should \ - --reporter $(REPORTER) \ - --timeout $(TIMEOUT) \ - $(MOCHA_OPTS) \ - $(TESTS) + @npm test -test-cov: - @rm -f coverage.html - @$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=html-cov > coverage.html - #@$(MAKE) test MOCHA_OPTS='--require blanket' REPORTER=travis-cov - @ls -lh coverage.html +test-cov: clean + @npm run cover + @npm run report clean: - rm -rf ./lib-cov coverage.html + rm -rf ./coverage ./.nyc_output -.PHONY: test-cov lib-cov test +.PHONY: test-cov test diff --git a/README.md b/README.md index 4873891b..c9b9835f 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ -# Qiniu Resource Storage SDK for Node.js +# Qiniu Cloud SDK for Node.js [![@qiniu on weibo](http://img.shields.io/badge/weibo-%40qiniutek-blue.svg)](http://weibo.com/qiniutek) [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg)](LICENSE.md) -[![Build Status](https://travis-ci.org/qiniu/nodejs-sdk.v6.png?branch=master)](https://travis-ci.org/qiniu/nodejs-sdk.v6) -[![Code Climate](https://codeclimate.com/github/qiniu/nodejs-sdk.png)](https://codeclimate.com/github/qiniu/nodejs-sdk) +[![NodeJS CI](https://github.com/qiniu/nodejs-sdk/actions/workflows/ci-test.yml/badge.svg?branch=master)](https://github.com/qiniu/nodejs-sdk/actions/workflows/ci-test.yml) +[![GitHub release](https://img.shields.io/github/v/tag/qiniu/nodejs-sdk.svg?label=release)](https://github.com/qiniu/nodejs-sdk/releases) +[![Code Climate](https://codeclimate.com/github/qiniu/nodejs-sdk.svg)](https://codeclimate.com/github/qiniu/nodejs-sdk) +[![Coverage Status](https://codecov.io/gh/qiniu/nodejs-sdk/branch/master/graph/badge.svg)](https://codecov.io/gh/qiniu/nodejs-sdk) [![Latest Stable Version](https://img.shields.io/npm/v/qiniu.svg)](https://www.npmjs.com/package/qiniu) ## 下载 @@ -18,7 +20,7 @@ $ npm install qiniu ### 从 release 版本下载 -下载地址：https://github.com/qiniu/nodejs-sdk/releases +下载地址：[https://github.com/qiniu/nodejs-sdk/releases](https://github.com/qiniu/nodejs-sdk/releases) 这里可以下载到旧版本的SDK，release 版本有版本号，有 [CHANGELOG](https://github.com/qiniu/nodejs-sdk/blob/master/CHANGELOG.md)，使用规格也会比较稳定。 @@ -30,6 +32,13 @@ $ npm install qiniu 参考文档：[七牛云存储 Node.js SDK 使用指南](http://developer.qiniu.com/kodo/sdk/nodejs) +## 测试 +``` +$ cd ./test/ +$ source test-env.sh +$ mocha --grep 'bucketinfo' +``` + ## 贡献代码 1. Fork diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..593ea60b --- /dev/null +++ b/codecov.yml @@ -0,0 +1,28 @@ +codecov: + ci: + - prow.qiniu.io # prow 里面运行需添加，其他 CI 不要 + require_ci_to_pass: no # 改为 no，否则 codecov 会等待其他 GitHub 上所有 CI 通过才会留言。 + +github_checks: #关闭github checks + annotations: false + +comment: + layout: "reach, diff, flags, files" + behavior: new # 默认是更新旧留言，改为 new，删除旧的，增加新的。 + require_changes: false # if true: only post the comment if coverage changes + require_base: no # [yes :: must have a base report to post] + require_head: yes # [yes :: must have a head report to post] + branches: # branch names that can post comment + - "master" + +coverage: + status: # 评判 pr 通过的标准 + patch: off + project: # project 统计所有代码x + default: + # basic + target: 80% # 总体通过标准 + threshold: 3% # 允许单次下降的幅度 + base: auto + if_not_found: success + if_ci_failed: error diff --git a/docs/nodejs-sdk-v7.md b/docs/nodejs-sdk-v7.md index b7772e99..128e3778 100644 --- a/docs/nodejs-sdk-v7.md +++ b/docs/nodejs-sdk-v7.md @@ -1,7 +1,7 @@ # 简介 -此 SDK 适用于 Node.js v4 及以上版本。使用此 SDK 构建您的网络应用程序，能让您以非常便捷的方式将数据安全地存储到七牛云上。无论您的网络应用是一个网站程序，还是包括从云端（服务端程序）到终端（手持设备应用）的架构服务和应用，通过七牛云及其 SDK，都能让您应用程序的终端用户高速上传和下载，同时也让您的服务端更加轻盈。 +此 SDK 适用于 Node.js v6 及以上版本。使用此 SDK 构建您的网络应用程序，能让您以非常便捷的方式将数据安全地存储到七牛云上。无论您的网络应用是一个网站程序，还是包括从云端（服务端程序）到终端（手持设备应用）的架构服务和应用，通过七牛云及其 SDK，都能让您应用程序的终端用户高速上传和下载，同时也让您的服务端更加轻盈。 Node.js SDK 属于七牛服务端SDK之一，主要有如下功能： @@ -16,7 +16,7 @@ Node.js SDK 属于七牛服务端SDK之一，主要有如下功能： - [Node.js SDK 项目地址](https://github.com/qiniu/nodejs-sdk) - [Node.js SDK 发布地址](https://github.com/qiniu/nodejs-sdk/releases) -- [Node.js SDK 历史文档](/kodo/sdk/nodejs-sdk-6) +- [Node.js SDK 历史文档](https://developer.qiniu.com/kodo/3828/node-js-v6) # 安装 @@ -58,7 +58,7 @@ $ npm install qiniu ### 上传流程 -七牛文件上传分为客户端上传（主要是指网页端和移动端等面向终端用户的场景）和服务端上传两种场景，具体可以参考文档[七牛业务流程](/kodo/manual/programming-model)。 +七牛文件上传分为客户端上传（主要是指网页端和移动端等面向终端用户的场景）和服务端上传两种场景，具体可以参考文档[七牛业务流程](https://developer.qiniu.com/kodo/1205/programming-model)。 服务端SDK在上传方面主要提供两种功能，一种是生成客户端上传所需要的上传凭证，另外一种是直接上传文件到云端。 @@ -89,7 +89,7 @@ var putPolicy = new qiniu.rs.PutPolicy(options); var uploadToken=putPolicy.uploadToken(mac); ``` -默认情况下，在不指定上传凭证的有效时间情况下，默认有效期为1个小时。也可以自行指定上传凭证的有效期，例如： +默认情况下，在不指定上传凭证的有效时间情况下，默认有效期为 1 个小时。也可以自行指定上传凭证的有效期，例如： ``` //自定义凭证有效期（示例2小时，expires单位为秒，为上传凭证的有效时间） @@ -124,7 +124,7 @@ var uploadToken=putPolicy.uploadToken(mac); {"hash":"Ftgm-CkWePC9fzMBTRNmPMhGBcSV","key":"qiniu.jpg"} ``` -有时候我们希望能自定义这个返回的JSON格式的内容，可以通过设置`returnBody`参数来实现，在`returnBody`中，我们可以使用七牛支持的[魔法变量](/kodo/manual/vars#magicvar)和[自定义变量](/kodo/manual/vars#xvar)。 +有时候我们希望能自定义这个返回的JSON格式的内容，可以通过设置`returnBody`参数来实现，在`returnBody`中，我们可以使用七牛支持的[魔法变量](https://developer.qiniu.com/kodo/1235/vars#magicvar)和[自定义变量](https://developer.qiniu.com/kodo/1235/vars#xvar)。 ``` var options = { @@ -194,7 +194,7 @@ var putPolicy = new qiniu.rs.PutPolicy(options); var uploadToken=putPolicy.uploadToken(mac); ``` -队列 pipeline 请参阅[创建私有队列](https://portal.qiniu.com/dora/create-mps)；转码操作具体参数请参阅[音视频转码](/dora/api/audio-and-video-transcoding-avthumb)；saveas 请参阅[处理结果另存](/dora/api/processing-results-save-saveas)。 +队列 pipeline 请参阅[创建私有队列](https://portal.qiniu.com/dora/create-mps)；转码操作具体参数请参阅[音视频转码](https://developer.qiniu.com/dora/1248/audio-and-video-transcoding-avthumb)；saveas 请参阅[处理结果另存](https://developer.qiniu.com/dora/1305/processing-results-save-saveas)。 #### 带自定义参数的凭证 @@ -243,14 +243,16 @@ config.zone = qiniu.zone.Zone_z0; ``` -其中关于`Zone`对象和机房的关系如下： +其中关于`Zone`对象和区域的关系如下： -|机房|Zone对象| -|---|-----| -|华东|`qiniu.zone.Zone_z0`| -|华北|`qiniu.zone.Zone_z1`| -|华南|`qiniu.zone.Zone_z2`| -|北美|`qiniu.zone.Zone_na0`| +| 区域 | Zone 对象 | +|--------------|-----------------------------| +| 华东-浙江 | `qiniu.zone.Zone_z0` | +| 华东-浙江2 | `qiniu.zone.Zone_cn_east_2` | +| 华北-河北 | `qiniu.zone.Zone_z1` | +| 华南-广东 | `qiniu.zone.Zone_z2` | +| 北美-洛杉矶 | `qiniu.zone.Zone_na0` | +| 亚太-新加坡（原东南亚） | `qiniu.zone.Zone_as0` | #### 文件上传（表单方式） @@ -408,7 +410,7 @@ console.log(publicDownloadUrl); ### 私有空间 -对于私有空间，首先需要按照公开空间的文件访问方式构建对应的公开空间访问链接，然后再对这个链接进行私有授权签名。 +对于私有空间，其访问链接需要进行签名才能访问，且有访问日期限制。因此需要额外传入过期时间戳。 ``` var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); @@ -1347,9 +1349,9 @@ console.log(finalUrl); # API 参考 -- [存储 API 参考](/kodo) -- [融合CDN API 参考](/fusion) -- [官方数据处理 API 参考](/dora) +- [存储 API 参考](https://developer.qiniu.com/kodo) +- [融合CDN API 参考](https://developer.qiniu.com/fusion) +- [官方数据处理 API 参考](https://developer.qiniu.com/dora) # 常见问题 @@ -1362,7 +1364,7 @@ console.log(finalUrl); 如果您有任何关于我们文档或产品的建议和想法，欢迎您通过以下方式与我们互动讨论： -* [技术论坛](http://segmentfault.com/qiniu) - 在这里您可以和其他开发者愉快的讨论如何更好的使用七牛云服务 +* [技术论坛](https://segmentfault.com/qiniu) - 在这里您可以和其他开发者愉快的讨论如何更好的使用七牛云服务 * [提交工单](https://support.qiniu.com/tickets/new) - 如果您的问题不适合在论坛讨论或希望及时解决，您也可以提交一个工单，我们的技术支持人员会第一时间回复您 * [博客](http://blog.qiniu.com) - 这里会持续更新发布市场活动和技术分享文章 * [微博](http://weibo.com/qiniutek) diff --git a/examples/atlab_check_qiniu_auth.js b/examples/atlab_check_qiniu_auth.js index 27ca4123..3c87528b 100644 --- a/examples/atlab_check_qiniu_auth.js +++ b/examples/atlab_check_qiniu_auth.js @@ -1,20 +1,24 @@ -const qiniu = require("../index.js"); -const proc = require("process"); +const qiniu = require('../index.js'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); -var reqURL = "http://serve.atlab.ai/v1/eval/facex-detect"; +var reqURL = 'http://serve.atlab.ai/v1/eval/facex-detect'; var contentType = 'application/json'; var reqBody = '{"data":{"uri":"https://ors35x6a7.qnssl.com/atshow-face-detection-20170703/1.png"}}'; var accessToken = qiniu.util.generateAccessTokenV2(mac, reqURL, 'POST', contentType, reqBody); var headers = { - 'Authorization': accessToken, - 'Content-Type': contentType, -} + Authorization: accessToken, + 'Content-Type': contentType +}; -qiniu.rpc.post(reqURL, reqBody, headers, function(err, body, info) { +qiniu.rpc.post(reqURL, reqBody, headers, function (err, body, info) { + if (err) { + console.error(err); + return; + } console.log(info); console.log(body); -}); \ No newline at end of file +}); diff --git a/examples/bucket_image_unimage.js b/examples/bucket_image_unimage.js index 6295a441..518a70b2 100644 --- a/examples/bucket_image_unimage.js +++ b/examples/bucket_image_unimage.js @@ -1,30 +1,29 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); var bucketManager = new qiniu.rs.BucketManager(mac, config); -var bucket = "if-pbl"; -var srcSiteUrl = "http://www.baidu.com/"; +var bucket = 'if-pbl'; +var srcSiteUrl = 'http://www.baidu.com/'; var srcHost = null; -bucketManager.image(bucket, srcSiteUrl, srcHost, function(err, respBody, - respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - console.log(respInfo.statusCode); - - //unimage - bucketManager.unimage(bucket, function(err1, respBody1, respInfo1) { - if (err1) { - throw err; - } - console.log(respInfo1.statusCode); - }); - } +bucketManager.image(bucket, srcSiteUrl, srcHost, function (err, respBody, + respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + console.log(respInfo.statusCode); + // unimage + bucketManager.unimage(bucket, function (err1, respBody1, respInfo1) { + if (err1) { + throw err; + } + console.log(respInfo1.statusCode); + }); + } }); diff --git a/examples/cdn_create_timestamp_antileech_url.js b/examples/cdn_create_timestamp_antileech_url.js index ec5ef915..f3f32fc3 100644 --- a/examples/cdn_create_timestamp_antileech_url.js +++ b/examples/cdn_create_timestamp_antileech_url.js @@ -1,17 +1,17 @@ -const qiniu = require("qiniu"); +const qiniu = require('qiniu'); var domain = 'http://sq.qiniuts.com'; -var fileName = "1491535764000.png"; -//加密密钥 +var fileName = '1491535764000.png'; +// 加密密钥 var encryptKey = '**'; -var query = "imageView2/2/w/480/format/jpg" +var query = 'imageView2/2/w/480/format/jpg'; var deadline = parseInt(Date.now() / 1000) + 3600; var cdnManager = new qiniu.cdn.CdnManager(null); var finalUrl = cdnManager.createTimestampAntiLeechUrl(domain, fileName, query, - encryptKey, deadline); + encryptKey, deadline); console.log(finalUrl); diff --git a/examples/cdn_get_bandwidth_data.js b/examples/cdn_get_bandwidth_data.js index 3fa88cf5..1071fecb 100644 --- a/examples/cdn_get_bandwidth_data.js +++ b/examples/cdn_get_bandwidth_data.js @@ -1,13 +1,13 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); -//域名列表 +// 域名列表 var domains = [ - 'if-pbl.qiniudn.com', - 'qdisk.qiniudn.com' + 'if-pbl.qiniudn.com', + 'qdisk.qiniudn.com' ]; -//指定日期 +// 指定日期 var startDate = '2017-06-20'; var endDate = '2017-06-22'; var granularity = 'day'; @@ -16,36 +16,36 @@ var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var cdnManager = new qiniu.cdn.CdnManager(mac); -//获取域名带宽 -cdnManager.getBandwidthData(startDate, endDate, granularity, domains, function( - err, respBody, respInfo) { - if (err) { - console.log(err); - throw err; - } +// 获取域名带宽 +cdnManager.getBandwidthData(startDate, endDate, granularity, domains, function ( + err, respBody, respInfo) { + if (err) { + console.log(err); + throw err; + } - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { - var jsonBody = JSON.parse(respBody); - var code = jsonBody.code; - console.log(code); + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { + var jsonBody = JSON.parse(respBody); + var code = jsonBody.code; + console.log(code); - var tickTime = jsonBody.time; - console.log(tickTime); + var tickTime = jsonBody.time; + console.log(tickTime); - var bandwidthData = jsonBody.data; - domains.forEach(function(domain) { - var bandwidthDataOfDomain = bandwidthData[domain]; - if (bandwidthDataOfDomain != null) { - console.log("bandwidth data for:" + domain); - var bandwidthChina = bandwidthDataOfDomain["china"]; - var bandwidthOversea = bandwidthDataOfDomain["oversea"]; - console.log(bandwidthChina); - console.log(bandwidthOversea); - } else { - console.log("no bandwidth data for:" + domain); - } - console.log("----------"); - }); - } + var bandwidthData = jsonBody.data; + domains.forEach(function (domain) { + var bandwidthDataOfDomain = bandwidthData[domain]; + if (bandwidthDataOfDomain != null) { + console.log('bandwidth data for:' + domain); + var bandwidthChina = bandwidthDataOfDomain.china; + var bandwidthOversea = bandwidthDataOfDomain.oversea; + console.log(bandwidthChina); + console.log(bandwidthOversea); + } else { + console.log('no bandwidth data for:' + domain); + } + console.log('----------'); + }); + } }); diff --git a/examples/cdn_get_flux_data.js b/examples/cdn_get_flux_data.js index 60af7b76..f0a361a2 100644 --- a/examples/cdn_get_flux_data.js +++ b/examples/cdn_get_flux_data.js @@ -1,13 +1,13 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); -//域名列表 +// 域名列表 var domains = [ - 'if-pbl.qiniudn.com', - 'qdisk.qiniudn.com' + 'if-pbl.qiniudn.com', + 'qdisk.qiniudn.com' ]; -//指定日期 +// 指定日期 var startDate = '2017-06-20'; var endDate = '2017-06-22'; var granularity = 'day'; @@ -16,35 +16,35 @@ var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var cdnManager = new qiniu.cdn.CdnManager(mac); -//获取域名流量 -cdnManager.getFluxData(startDate, endDate, granularity, domains, function(err, - respBody, respInfo) { - if (err) { - throw err; - } +// 获取域名流量 +cdnManager.getFluxData(startDate, endDate, granularity, domains, function (err, + respBody, respInfo) { + if (err) { + throw err; + } - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { - var jsonBody = JSON.parse(respBody); - var code = jsonBody.code; - console.log(code); + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { + var jsonBody = JSON.parse(respBody); + var code = jsonBody.code; + console.log(code); - var tickTime = jsonBody.time; - console.log(tickTime); + var tickTime = jsonBody.time; + console.log(tickTime); - var fluxData = jsonBody.data; - domains.forEach(function(domain) { - var fluxDataOfDomain = fluxData[domain]; - if (fluxDataOfDomain != null) { - console.log("flux data for:" + domain); - var fluxChina = fluxDataOfDomain["china"]; - var fluxOversea = fluxDataOfDomain["oversea"]; - console.log(fluxChina); - console.log(fluxOversea); - } else { - console.log("no flux data for:" + domain); - } - console.log("----------"); - }); - } + var fluxData = jsonBody.data; + domains.forEach(function (domain) { + var fluxDataOfDomain = fluxData[domain]; + if (fluxDataOfDomain != null) { + console.log('flux data for:' + domain); + var fluxChina = fluxDataOfDomain.china; + var fluxOversea = fluxDataOfDomain.oversea; + console.log(fluxChina); + console.log(fluxOversea); + } else { + console.log('no flux data for:' + domain); + } + console.log('----------'); + }); + } }); diff --git a/examples/cdn_get_log_list.js b/examples/cdn_get_log_list.js index 0cf22f4a..ab087314 100644 --- a/examples/cdn_get_log_list.js +++ b/examples/cdn_get_log_list.js @@ -1,27 +1,27 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); -//域名列表 +// 域名列表 var domains = [ - 'if-pbl.qiniudn.com', - 'qdisk.qiniudn.com' + 'if-pbl.qiniudn.com', + 'qdisk.qiniudn.com' ]; -//指定日期 +// 指定日期 var logDay = '2017-06-20'; var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var cdnManager = new qiniu.cdn.CdnManager(mac); -//获取域名日志 -cdnManager.getCdnLogList(domains, logDay, function(err, respBody, respInfo) { - if (err) { - throw err; - } +// 获取域名日志 +cdnManager.getCdnLogList(domains, logDay, function (err, respBody, respInfo) { + if (err) { + throw err; + } - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { /** { "code":0, @@ -38,22 +38,22 @@ cdnManager.getCdnLogList(domains, logDay, function(err, respBody, respInfo) { } } */ - var jsonBody = JSON.parse(respBody); - var code = jsonBody.code; - console.log(code); - var logData = jsonBody.data; - domains.forEach(function(domain) { - console.log("log for domain: " + domain); - var domainLogs = logData[domain]; - if (domainLogs != null) { - domainLogs.forEach(function(logItem) { - console.log(logItem.name); - console.log(logItem.size); - console.log(logItem.mtime); - console.log(logItem.url); + var jsonBody = JSON.parse(respBody); + var code = jsonBody.code; + console.log(code); + var logData = jsonBody.data; + domains.forEach(function (domain) { + console.log('log for domain: ' + domain); + var domainLogs = logData[domain]; + if (domainLogs != null) { + domainLogs.forEach(function (logItem) { + console.log(logItem.name); + console.log(logItem.size); + console.log(logItem.mtime); + console.log(logItem.url); + }); + console.log('------------------'); + } }); - console.log("------------------"); - } - }); - } + } }); diff --git a/examples/cdn_prefetch_urls.js b/examples/cdn_prefetch_urls.js index 3ec9157b..7afb8507 100644 --- a/examples/cdn_prefetch_urls.js +++ b/examples/cdn_prefetch_urls.js @@ -1,33 +1,32 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); - -//URL 列表 +// URL 列表 var urlsToPrefetch = [ - 'http://if-pbl.qiniudn.com/nodejs.png', - 'http://if-pbl.qiniudn.com/qiniu.jpg' + 'http://if-pbl.qiniudn.com/nodejs.png', + 'http://if-pbl.qiniudn.com/qiniu.jpg' ]; var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var cdnManager = new qiniu.cdn.CdnManager(mac); -//预取链接 -cdnManager.prefetchUrls(urlsToPrefetch, function(err, respBody, respInfo) { - if (err) { - throw err; - } +// 预取链接 +cdnManager.prefetchUrls(urlsToPrefetch, function (err, respBody, respInfo) { + if (err) { + throw err; + } - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { - var jsonBody = JSON.parse(respBody); - console.log(jsonBody.code); - console.log(jsonBody.error); - console.log(jsonBody.requestId); - console.log(jsonBody.invalidUrls); - console.log(jsonBody.invalidDirs); - console.log(jsonBody.urlQuotaDay); - console.log(jsonBody.urlSurplusDay); - console.log(jsonBody.dirQuotaDay); - console.log(jsonBody.dirSurplusDay); - } + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { + var jsonBody = JSON.parse(respBody); + console.log(jsonBody.code); + console.log(jsonBody.error); + console.log(jsonBody.requestId); + console.log(jsonBody.invalidUrls); + console.log(jsonBody.invalidDirs); + console.log(jsonBody.urlQuotaDay); + console.log(jsonBody.urlSurplusDay); + console.log(jsonBody.dirQuotaDay); + console.log(jsonBody.dirSurplusDay); + } }); diff --git a/examples/cdn_refresh_urls_dirs.js b/examples/cdn_refresh_urls_dirs.js index 1a38cf65..2dd7bf65 100644 --- a/examples/cdn_refresh_urls_dirs.js +++ b/examples/cdn_refresh_urls_dirs.js @@ -1,84 +1,82 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); -//URL 列表 +// URL 列表 var urlsToRefresh = [ - 'http://if-pbl.qiniudn.com/nodejs.png', - 'http://if-pbl.qiniudn.com/qiniu.jpg' + 'http://if-pbl.qiniudn.com/nodejs.png', + 'http://if-pbl.qiniudn.com/qiniu.jpg' ]; -//DIR 列表 +// DIR 列表 var dirsToRefresh = [ - 'http://if-pbl.qiniudn.com/examples/', - 'http://if-pbl.qiniudn.com/images/' + 'http://if-pbl.qiniudn.com/examples/', + 'http://if-pbl.qiniudn.com/images/' ]; var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var cdnManager = new qiniu.cdn.CdnManager(mac); -//刷新链接 -cdnManager.refreshUrls(urlsToRefresh, function(err, respBody, respInfo) { - if (err) { - throw err; - } - - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { - var jsonBody = JSON.parse(respBody); - console.log(jsonBody.code); - console.log(jsonBody.error); - console.log(jsonBody.requestId); - console.log(jsonBody.invalidUrls); - console.log(jsonBody.invalidDirs); - console.log(jsonBody.urlQuotaDay); - console.log(jsonBody.urlSurplusDay); - console.log(jsonBody.dirQuotaDay); - console.log(jsonBody.dirSurplusDay); - } - +// 刷新链接 +cdnManager.refreshUrls(urlsToRefresh, function (err, respBody, respInfo) { + if (err) { + throw err; + } + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { + var jsonBody = JSON.parse(respBody); + console.log(jsonBody.code); + console.log(jsonBody.error); + console.log(jsonBody.requestId); + console.log(jsonBody.invalidUrls); + console.log(jsonBody.invalidDirs); + console.log(jsonBody.urlQuotaDay); + console.log(jsonBody.urlSurplusDay); + console.log(jsonBody.dirQuotaDay); + console.log(jsonBody.dirSurplusDay); + } }); -//刷新目录，刷新目录需要联系七牛技术支持开通权限 -cdnManager.refreshDirs(dirsToRefresh, function(err, respBody, respInfo) { - if (err) { - throw err; - } +// 刷新目录，刷新目录需要联系七牛技术支持开通权限 +cdnManager.refreshDirs(dirsToRefresh, function (err, respBody, respInfo) { + if (err) { + throw err; + } - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { - var jsonBody = JSON.parse(respBody); - console.log(jsonBody.code); - console.log(jsonBody.error); - console.log(jsonBody.requestId); - console.log(jsonBody.invalidUrls); - console.log(jsonBody.invalidDirs); - console.log(jsonBody.urlQuotaDay); - console.log(jsonBody.urlSurplusDay); - console.log(jsonBody.dirQuotaDay); - console.log(jsonBody.dirSurplusDay); - } + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { + var jsonBody = JSON.parse(respBody); + console.log(jsonBody.code); + console.log(jsonBody.error); + console.log(jsonBody.requestId); + console.log(jsonBody.invalidUrls); + console.log(jsonBody.invalidDirs); + console.log(jsonBody.urlQuotaDay); + console.log(jsonBody.urlSurplusDay); + console.log(jsonBody.dirQuotaDay); + console.log(jsonBody.dirSurplusDay); + } }); -//一起刷新 -cdnManager.refreshUrlsAndDirs(urlsToRefresh, dirsToRefresh, function(err, - respBody, respInfo) { - if (err) { - throw err; - } +// 一起刷新 +cdnManager.refreshUrlsAndDirs(urlsToRefresh, dirsToRefresh, function (err, + respBody, respInfo) { + if (err) { + throw err; + } - console.log(respInfo.statusCode); - if (respInfo.statusCode == 200) { - var jsonBody = JSON.parse(respBody); - console.log(jsonBody.code); - console.log(jsonBody.error); - console.log(jsonBody.requestId); - console.log(jsonBody.invalidUrls); - console.log(jsonBody.invalidDirs); - console.log(jsonBody.urlQuotaDay); - console.log(jsonBody.urlSurplusDay); - console.log(jsonBody.dirQuotaDay); - console.log(jsonBody.dirSurplusDay); - } + console.log(respInfo.statusCode); + if (respInfo.statusCode == 200) { + var jsonBody = JSON.parse(respBody); + console.log(jsonBody.code); + console.log(jsonBody.error); + console.log(jsonBody.requestId); + console.log(jsonBody.invalidUrls); + console.log(jsonBody.invalidDirs); + console.log(jsonBody.urlQuotaDay); + console.log(jsonBody.urlSurplusDay); + console.log(jsonBody.dirQuotaDay); + console.log(jsonBody.dirSurplusDay); + } }); diff --git a/examples/create_uptoken.js b/examples/create_uptoken.js index d4bbd495..e9005f4f 100644 --- a/examples/create_uptoken.js +++ b/examples/create_uptoken.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -7,69 +7,68 @@ var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var bucket = proc.env.QINIU_TEST_BUCKET; -//简单上传凭证 +// 简单上传凭证 var options = { - scope: bucket, + scope: bucket }; var putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); -//自定义凭证有效期（示例2小时） -var options = { - scope: bucket, - expires: 7200 -} -var putPolicy = new qiniu.rs.PutPolicy(options); +// 自定义凭证有效期（示例2小时） +options = { + scope: bucket, + expires: 7200 +}; +putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); // 覆盖上传凭证 var keyToOverwrite = 'qiniu.mp4'; -var options = { - scope: bucket + ":" + keyToOverwrite -} -var putPolicy = new qiniu.rs.PutPolicy(options); +options = { + scope: bucket + ':' + keyToOverwrite +}; +putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); -//自定义上传回复（非callback模式）凭证 -var options = { - scope: bucket, - returnBody: '{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"bucket":"$(bucket)","name":"$(x:name)"}' -} -var putPolicy = new qiniu.rs.PutPolicy(options); +// 自定义上传回复（非callback模式）凭证 +options = { + scope: bucket, + returnBody: '{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"bucket":"$(bucket)","name":"$(x:name)"}' +}; +putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); -//带回调业务服务器的凭证（application/json） -var options = { - scope: bucket, - callbackUrl: 'http://api.example.com/qiniu/upload/callback', - callbackBody: '{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"bucket":"$(bucket)","name":"$(x:name)"}', - callbackBodyType: 'application/json' -} -var putPolicy = new qiniu.rs.PutPolicy(options); +// 带回调业务服务器的凭证（application/json） +options = { + scope: bucket, + callbackUrl: 'http://api.example.com/qiniu/upload/callback', + callbackBody: '{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"bucket":"$(bucket)","name":"$(x:name)"}', + callbackBodyType: 'application/json' +}; +putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); - -//带回调业务服务器的凭证（application/x-www-form-urlencoded） -var options = { - scope: bucket, - callbackUrl: 'http://api.example.com/qiniu/upload/callback', - callbackBody: 'key=$(key)&hash=$(etag)&bucket=$(bucket)&fsize=$(fsize)&name=$(x:name)' -} -var putPolicy = new qiniu.rs.PutPolicy(options); +// 带回调业务服务器的凭证（application/x-www-form-urlencoded） +options = { + scope: bucket, + callbackUrl: 'http://api.example.com/qiniu/upload/callback', + callbackBody: 'key=$(key)&hash=$(etag)&bucket=$(bucket)&fsize=$(fsize)&name=$(x:name)' +}; +putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); -//带数据处理的凭证 +// 带数据处理的凭证 var saveMp4Entry = qiniu.util.urlsafeBase64Encode(bucket + - ":avthumb_test_target.mp4"); + ':avthumb_test_target.mp4'); var saveJpgEntry = qiniu.util.urlsafeBase64Encode(bucket + - ":vframe_test_target.jpg"); -var avthumbMp4Fop = "avthumb/mp4|saveas/" + saveMp4Entry; -var vframeJpgFop = "vframe/jpg/offset/1|saveas/" + saveJpgEntry; -var options = { - scope: bucket, - persistentOps: avthumbMp4Fop + ";" + vframeJpgFop, - persistentPipeline: "video-pipe", - persistentNotifyUrl: "http://api.example.com/qiniu/pfop/notify", -} -var putPolicy = new qiniu.rs.PutPolicy(options); + ':vframe_test_target.jpg'); +var avthumbMp4Fop = 'avthumb/mp4|saveas/' + saveMp4Entry; +var vframeJpgFop = 'vframe/jpg/offset/1|saveas/' + saveJpgEntry; +options = { + scope: bucket, + persistentOps: avthumbMp4Fop + ';' + vframeJpgFop, + persistentPipeline: 'video-pipe', + persistentNotifyUrl: 'http://api.example.com/qiniu/pfop/notify' +}; +putPolicy = new qiniu.rs.PutPolicy(options); console.log(putPolicy.uploadToken(mac)); diff --git a/examples/form_qvm_upload.js b/examples/form_qvm_upload.js new file mode 100644 index 00000000..ab0a9c2e --- /dev/null +++ b/examples/form_qvm_upload.js @@ -0,0 +1,71 @@ +const qiniu = require('../index.js'); +const proc = require('process'); + +var bucket = proc.env.QINIU_TEST_BUCKET; +var accessKey = proc.env.QINIU_ACCESS_KEY; +var secretKey = proc.env.QINIU_SECRET_KEY; +var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); +var options = { + scope: bucket +}; +var putPolicy = new qiniu.rs.PutPolicy(options); + +var uploadToken = putPolicy.uploadToken(mac); +var config = new qiniu.conf.Config(); +var localFile = '/Users/jemy/Documents/qiniu.mp4'; +// config.zone = qiniu.zone.Zone_z0; + +// construct a new zone +// 华东 +var ZONE_QVM_Z0 = new qiniu.conf.Zone([ + 'free-qvm-z0-xs.qiniup.com' +], [ + 'free-qvm-z0-xs.qiniup.com' +], 'iovip.qbox.me', +'rs.qbox.me', +'rsf.qbox.me', +'api.qiniuapi.com'); + +// 华北 +var ZONE_QVM_Z1 = new qiniu.conf.Zone([ + 'free-qvm-z1-zz.qiniup.com' +], [ + 'free-qvm-z1-zz.qiniup.com' +], 'iovip-z1.qbox.me', +'rs-z1.qbox.me', +'rsf-z1.qbox.me', +'api-z1.qiniuapi.com'); + +config.zone = ZONE_QVM_Z0; +config.zone = ZONE_QVM_Z1; +var formUploader = new qiniu.form_up.FormUploader(config); +var putExtra = new qiniu.form_up.PutExtra(); +// bytes +formUploader.put(uploadToken, null, 'hello', null, function (respErr, + respBody, respInfo) { + if (respErr) { + throw respErr; + } + + if (respInfo.statusCode == 200) { + console.log(respBody); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } +}); + +// file +formUploader.putFile(uploadToken, null, localFile, putExtra, function (respErr, + respBody, respInfo) { + if (respErr) { + throw respErr; + } + + if (respInfo.statusCode == 200) { + console.log(respBody); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } +}); diff --git a/examples/form_upload_simple.js b/examples/form_upload_simple.js index dd2b5964..4570269d 100644 --- a/examples/form_upload_simple.js +++ b/examples/form_upload_simple.js @@ -1,48 +1,43 @@ -const qiniu = require("../index.js"); -const proc = require("process"); +const os = require('os'); -var bucket = proc.env.QINIU_TEST_BUCKET; -var accessKey = proc.env.QINIU_ACCESS_KEY; -var secretKey = proc.env.QINIU_SECRET_KEY; -var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); -var options = { - scope: bucket, -} -var putPolicy = new qiniu.rs.PutPolicy(options); +const qiniu = require('qiniu'); -var uploadToken = putPolicy.uploadToken(mac); -var config = new qiniu.conf.Config(); -var localFile = "/Users/jemy/Documents/qiniu.mp4"; -//config.zone = qiniu.zone.Zone_z0; -var formUploader = new qiniu.form_up.FormUploader(config); -var putExtra = new qiniu.form_up.PutExtra(); +const bucket = process.env.QINIU_TEST_BUCKET; +const accessKey = process.env.QINIU_ACCESS_KEY; +const secretKey = process.env.QINIU_SECRET_KEY; +const mac = new qiniu.auth.digest.Mac(accessKey, secretKey); +const options = { + scope: bucket +}; +const putPolicy = new qiniu.rs.PutPolicy(options); -//bytes -formUploader.put(uploadToken, null, "hello", null, function(respErr, - respBody, respInfo) { - if (respErr) { - throw respErr; - } +const uploadToken = putPolicy.uploadToken(mac); +const config = new qiniu.conf.Config(); +const localFile = os.homedir() + '/Downloads/83eda6926b94bb14.css'; +// config.zone = qiniu.zone.Zone_z0; +const formUploader = new qiniu.form_up.FormUploader(config); +const putExtra = new qiniu.form_up.PutExtra(); +// file +// putExtra.fname = 'frontend-static-resource/widgets/_next/static/css/83eda6926b94bb14.css'; +// putExtra.metadata = { +// 'x-qn-meta-name': 'qiniu' +// }; +formUploader.putFile( + uploadToken, + 'frontend-static-resource/widgets/_next/static/css/83eda6926b94bb14.css', + localFile, + putExtra, + function (respErr, + respBody, respInfo) { + if (respErr) { + throw respErr; + } - if (respInfo.statusCode == 200) { - console.log(respBody); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } -}); - -//file -formUploader.putFile(uploadToken, null, localFile, putExtra, function(respErr, - respBody, respInfo) { - if (respErr) { - throw respErr; - } - - if (respInfo.statusCode == 200) { - console.log(respBody); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } -}); + if (respInfo.statusCode === 200) { + console.log(respBody); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } + } +); diff --git a/examples/http_https_proxy.js b/examples/http_https_proxy.js index 9e2b3c60..4cc88618 100644 --- a/examples/http_https_proxy.js +++ b/examples/http_https_proxy.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); const tunnel = require('tunnel-agent'); var bucket = 'if-pbl'; @@ -7,45 +7,45 @@ var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var options = { - scope: bucket, -} + scope: bucket +}; var putPolicy = new qiniu.rs.PutPolicy(options); var uploadToken = putPolicy.uploadToken(mac); var config = new qiniu.conf.Config(); -//config.zone = qiniu.zone.Zone_z0; -//config.useHttpsDomain = true; +// config.zone = qiniu.zone.Zone_z0; +// config.useHttpsDomain = true; var formUploader = new qiniu.form_up.FormUploader(config); var putExtra = new qiniu.form_up.PutExtra(); -//设置HTTP(s)代理服务器，这里可以参考：https://github.com/request/tunnel-agent -//有几个方法: -//exports.httpOverHttp = httpOverHttp -//exports.httpsOverHttp = httpsOverHttp -//exports.httpOverHttps = httpOverHttps -//exports.httpsOverHttps = httpsOverHttps +// 设置HTTP(s)代理服务器，这里可以参考：https://github.com/request/tunnel-agent +// 有几个方法: +// exports.httpOverHttp = httpOverHttp +// exports.httpsOverHttp = httpsOverHttp +// exports.httpOverHttps = httpOverHttps +// exports.httpsOverHttps = httpsOverHttps var proxyAgent = tunnel.httpOverHttp({ - proxy: { - host: 'localhost', - port: 8888 - } + proxy: { + host: 'localhost', + port: 8888 + } }); qiniu.conf.RPC_HTTP_AGENT = proxyAgent; -//qiniu.conf.RPC_HTTPS_AGENT = proxyAgent; -//以代理方式上传 -formUploader.put(uploadToken, null, "hello", null, function(respErr, - respBody, respInfo) { - if (respErr) { - throw respErr; - } +// qiniu.conf.RPC_HTTPS_AGENT = proxyAgent; +// 以代理方式上传 +formUploader.put(uploadToken, null, 'hello', putExtra, function (respErr, + respBody, respInfo) { + if (respErr) { + throw respErr; + } - if (respInfo.statusCode == 200) { - console.log(respBody); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } + if (respInfo.statusCode == 200) { + console.log(respBody); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/object_lifecycle.js b/examples/object_lifecycle.js new file mode 100644 index 00000000..c2361b6d --- /dev/null +++ b/examples/object_lifecycle.js @@ -0,0 +1,30 @@ +const qiniu = require('../index'); +const proc = require('process'); + +const accessKey = proc.env.QINIU_ACCESS_KEY; +const secretKey = proc.env.QINIU_SECRET_KEY; +const mac = new qiniu.auth.digest.Mac(accessKey, secretKey); +const config = new qiniu.conf.Config(); +config.useHttpsDomain = true; +// config.zone = qiniu.zone.Zone_z0; +const bucketManager = new qiniu.rs.BucketManager(mac, config); +const bucket = proc.env.QINIU_TEST_BUCKET; +const key = 'test_file'; + +bucketManager.setObjectLifeCycle( + bucket, + key, + { + toIaAfterDays: 10, + toArchiveAfterDays: 20, + toDeepArchiveAfterDays: 30, + deleteAfterDays: 40 + }, + function (err, respBody, respInfo) { + if (err) { + console.log(err); + console.log(respInfo); + } + console.log(respBody); + } +); diff --git a/examples/pfops_video_plus.js b/examples/pfops_video_plus.js new file mode 100644 index 00000000..67648ecf --- /dev/null +++ b/examples/pfops_video_plus.js @@ -0,0 +1,62 @@ +var qiniu = require('qiniu'); +var urllib = require('urllib'); + +qiniu.conf.ACCESS_KEY = 'ak'; +qiniu.conf.SECRET_KEY = 'sk'; + +var url = 'http://argus.atlab.ai/v1/video/89999sssss'; + +var mac = new qiniu.auth.digest.Mac(qiniu.conf.ACCESS_KEY, qiniu.conf.SECRET_KEY); + +var json = { + + data: { + uri: 'http://test.qiniu.com/Videos/2016-09/39/9d019f7acab742ddbc5f4db02b6f72cb.mp4' + }, + params: { + async: false, + vframe: { + mode: 0, + interval: 5 + } + }, + ops: [ + { + op: 'pulp', + params: { + labels: [ + { + label: '1', + select: 2, + score: 0.0002 + }, + { + label: '2', + select: 2, + score: 0.0002 + } + + ] + } + } + ] +}; + +var accessToken = qiniu.util.generateAccessTokenV2(mac, url, 'POST', 'application/json', JSON.stringify(json)); + +urllib.request(url, { + method: 'POST', + headers: { + Authorization: accessToken, + 'Content-Type': 'application/json' + }, + data: JSON.stringify(json) +}, function (err, data, res) { + if (err) { + console.log(err); + throw err; // you need to handle error + } + console.log(res.statusCode); + console.log(res); + console.log(data.toString()); +}); diff --git a/examples/prefop.js b/examples/prefop.js index cd944328..8d330cd3 100644 --- a/examples/prefop.js +++ b/examples/prefop.js @@ -1,31 +1,31 @@ -const qiniu = require("qiniu"); +const qiniu = require('qiniu'); -//var persistentId = 'z0.594b66f745a2650c99aa9e57'; +// var persistentId = 'z0.594b66f745a2650c99aa9e57'; var persistentId = 'na0.58df4eee92129336c2075195'; var config = new qiniu.conf.Config(); config.useHttpsDomain = true; var operManager = new qiniu.fop.OperationManager(null, config); -//持久化数据处理返回的是任务的persistentId，可以根据这个id查询处理状态 -operManager.prefop(persistentId, function(err, respBody, respInfo) { - if (err) { - console.log(err); - throw err; - } +// 持久化数据处理返回的是任务的persistentId，可以根据这个id查询处理状态 +operManager.prefop(persistentId, function (err, respBody, respInfo) { + if (err) { + console.log(err); + throw err; + } - if (respInfo.statusCode == 200) { - console.log(respBody.inputBucket); - console.log(respBody.inputKey); - console.log(respBody.pipeline); - console.log(respBody.reqid); - respBody.items.forEach(function(item) { - console.log(item.cmd); - console.log(item.code); - console.log(item.desc); - console.log(item.hash); - console.log(item.key); - }); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } + if (respInfo.statusCode == 200) { + console.log(respBody.inputBucket); + console.log(respBody.inputKey); + console.log(respBody.pipeline); + console.log(respBody.reqid); + respBody.items.forEach(function (item) { + console.log(item.cmd); + console.log(item.code); + console.log(item.desc); + console.log(item.hash); + console.log(item.key); + }); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/resume_upload_simple.js b/examples/resume_upload_simple.js index 897c8d5c..67703488 100644 --- a/examples/resume_upload_simple.js +++ b/examples/resume_upload_simple.js @@ -1,43 +1,46 @@ -const qiniu = require("../index.js"); -const proc = require("process"); +const qiniu = require('../index.js'); +const proc = require('process'); var bucket = proc.env.QINIU_TEST_BUCKET; var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var options = { - scope: bucket, -} + scope: bucket +}; var putPolicy = new qiniu.rs.PutPolicy(options); var uploadToken = putPolicy.uploadToken(mac); var config = new qiniu.conf.Config(); -var localFile = "/Users/jemy/Documents/qiniu.mp4"; +var localFile = '/Users/jemy/Documents/qiniu.mp4'; config.zone = qiniu.zone.Zone_z0; config.useCdnDomain = true; var resumeUploader = new qiniu.resume_up.ResumeUploader(config); var putExtra = new qiniu.resume_up.PutExtra(); putExtra.params = { - "x:name": "", - "x:age": 27, -} + 'x:name': '', + 'x:age': 27 +}; +putExtra.metadata = { + 'x-qn-meta-name': 'qiniu' +}; putExtra.fname = 'testfile.mp4'; putExtra.resumeRecordFile = 'progress.log'; -putExtra.progressCallback = function(uploadBytes, totalBytes) { - console.log("progress:" + uploadBytes + "(" + totalBytes + ")"); -} +putExtra.progressCallback = function (uploadBytes, totalBytes) { + console.log('progress:' + uploadBytes + '(' + totalBytes + ')'); +}; -//file -resumeUploader.putFile(uploadToken, null, localFile, putExtra, function(respErr, - respBody, respInfo) { - if (respErr) { - throw respErr; - } +// file +resumeUploader.putFile(uploadToken, null, localFile, putExtra, function (respErr, + respBody, respInfo) { + if (respErr) { + throw respErr; + } - if (respInfo.statusCode == 200) { - console.log(respBody); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } + if (respInfo.statusCode == 200) { + console.log(respBody); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_batch_change_type.js b/examples/rs_batch_change_type.js index 4cb7961d..ceb47331 100644 --- a/examples/rs_batch_change_type.js +++ b/examples/rs_batch_change_type.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -9,31 +9,31 @@ var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var changeTypeOperations = [ - qiniu.rs.changeTypeOp(srcBucket, 'qiniu1.mp4', 1), - qiniu.rs.changeTypeOp(srcBucket, 'qiniu2.mp4', 1), - qiniu.rs.changeTypeOp(srcBucket, 'qiniu3.mp4', 1), - qiniu.rs.changeTypeOp(srcBucket, 'qiniu4.mp4', 1), + qiniu.rs.changeTypeOp(srcBucket, 'qiniu1.mp4', 1), + qiniu.rs.changeTypeOp(srcBucket, 'qiniu2.mp4', 1), + qiniu.rs.changeTypeOp(srcBucket, 'qiniu3.mp4', 1), + qiniu.rs.changeTypeOp(srcBucket, 'qiniu4.mp4', 1) ]; -bucketManager.batch(changeTypeOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(changeTypeOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log("success"); + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log('success'); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.statusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.statusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_batch_chgm.js b/examples/rs_batch_chgm.js index c3bfad35..aa3aab4a 100644 --- a/examples/rs_batch_chgm.js +++ b/examples/rs_batch_chgm.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -9,31 +9,31 @@ var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var chgmOperations = [ - qiniu.rs.changeMimeOp(srcBucket, 'qiniu1.mp4', 'video/x-mp4'), - qiniu.rs.changeMimeOp(srcBucket, 'qiniu2.mp4', 'video/x-mp4'), - qiniu.rs.changeMimeOp(srcBucket, 'qiniu3.mp4', 'video/x-mp4'), - qiniu.rs.changeMimeOp(srcBucket, 'qiniu4.mp4', 'video/x-mp4'), + qiniu.rs.changeMimeOp(srcBucket, 'qiniu1.mp4', 'video/x-mp4'), + qiniu.rs.changeMimeOp(srcBucket, 'qiniu2.mp4', 'video/x-mp4'), + qiniu.rs.changeMimeOp(srcBucket, 'qiniu3.mp4', 'video/x-mp4'), + qiniu.rs.changeMimeOp(srcBucket, 'qiniu4.mp4', 'video/x-mp4') ]; -bucketManager.batch(chgmOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(chgmOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log("success"); + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log('success'); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.statusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.statusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_batch_copy.js b/examples/rs_batch_copy.js index e8803638..2d049530 100644 --- a/examples/rs_batch_copy.js +++ b/examples/rs_batch_copy.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -11,33 +11,33 @@ var srcBucket = proc.env.QINIU_TEST_BUCKET; var srcKey = 'qiniu.mp4'; var destBucket = srcBucket; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var copyOperations = [ - qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu1.mp4', { - force: true - }), - qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu2.mp4'), - qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu3.mp4'), - qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu4.mp4'), + qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu1.mp4', { + force: true + }), + qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu2.mp4'), + qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu3.mp4'), + qiniu.rs.copyOp(srcBucket, srcKey, destBucket, 'qiniu4.mp4') ]; -bucketManager.batch(copyOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(copyOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log(item.code + "\tsuccess"); + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log(item.code + '\tsuccess'); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.deleteusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.deleteusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_batch_delete.js b/examples/rs_batch_delete.js index abf73fc7..8e6e218c 100644 --- a/examples/rs_batch_delete.js +++ b/examples/rs_batch_delete.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -9,31 +9,31 @@ var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var deleteOperations = [ - qiniu.rs.deleteOp(srcBucket, 'qiniu1.mp4'), - qiniu.rs.deleteOp(srcBucket, 'qiniu2.mp4'), - qiniu.rs.deleteOp(srcBucket, 'qiniu3.mp4'), - qiniu.rs.deleteOp(srcBucket, 'qiniu4x.mp4'), + qiniu.rs.deleteOp(srcBucket, 'qiniu1.mp4'), + qiniu.rs.deleteOp(srcBucket, 'qiniu2.mp4'), + qiniu.rs.deleteOp(srcBucket, 'qiniu3.mp4'), + qiniu.rs.deleteOp(srcBucket, 'qiniu4x.mp4') ]; -bucketManager.batch(deleteOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(deleteOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log(item.code + "\tsuccess"); + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log(item.code + '\tsuccess'); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.deleteusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.deleteusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_batch_delete_after_days.js b/examples/rs_batch_delete_after_days.js index 4f40118e..53d3e023 100644 --- a/examples/rs_batch_delete_after_days.js +++ b/examples/rs_batch_delete_after_days.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -9,31 +9,31 @@ var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var deleteAfterDaysOperations = [ - qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu1.mp4', 10), - qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu2.mp4', 10), - qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu3.mp4', 10), - qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu4.mp4', 10), + qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu1.mp4', 10), + qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu2.mp4', 10), + qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu3.mp4', 10), + qiniu.rs.deleteAfterDaysOp(srcBucket, 'qiniu4.mp4', 10) ]; -bucketManager.batch(deleteAfterDaysOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(deleteAfterDaysOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log("success"); + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log('success'); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.statusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.statusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_batch_move.js b/examples/rs_batch_move.js index 69c97727..b262456e 100644 --- a/examples/rs_batch_move.js +++ b/examples/rs_batch_move.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -10,31 +10,31 @@ var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; var destBucket = srcBucket; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var moveOperations = [ - qiniu.rs.moveOp(srcBucket, 'qiniu1.mp4', destBucket, 'qiniu1_move.mp4'), - qiniu.rs.moveOp(srcBucket, 'qiniu2.mp4', destBucket, 'qiniu2_move.mp4'), - qiniu.rs.moveOp(srcBucket, 'qiniu3.mp4', destBucket, 'qiniu3_move.mp4'), - qiniu.rs.moveOp(srcBucket, 'qiniu4.mp4', destBucket, 'qiniu4_move.mp4'), + qiniu.rs.moveOp(srcBucket, 'qiniu1.mp4', destBucket, 'qiniu1_move.mp4'), + qiniu.rs.moveOp(srcBucket, 'qiniu2.mp4', destBucket, 'qiniu2_move.mp4'), + qiniu.rs.moveOp(srcBucket, 'qiniu3.mp4', destBucket, 'qiniu3_move.mp4'), + qiniu.rs.moveOp(srcBucket, 'qiniu4.mp4', destBucket, 'qiniu4_move.mp4') ]; -bucketManager.batch(moveOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(moveOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log(item.code + "\tsuccess"); + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log(item.code + '\tsuccess'); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.deleteusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.deleteusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_batch_stat.js b/examples/rs_batch_stat.js index 0acd5a6d..163810f8 100644 --- a/examples/rs_batch_stat.js +++ b/examples/rs_batch_stat.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -9,33 +9,33 @@ var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; -//每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 +// 每个operations的数量不可以超过1000个，如果总数量超过1000，需要分批发送 var statOperations = [ - qiniu.rs.statOp(srcBucket, 'qiniu1.mp4'), - qiniu.rs.statOp(srcBucket, 'qiniu2.mp4'), - qiniu.rs.statOp(srcBucket, 'qiniu3.mp4'), - qiniu.rs.statOp(srcBucket, 'qiniu4x.mp4'), + qiniu.rs.statOp(srcBucket, 'qiniu1.mp4'), + qiniu.rs.statOp(srcBucket, 'qiniu2.mp4'), + qiniu.rs.statOp(srcBucket, 'qiniu3.mp4'), + qiniu.rs.statOp(srcBucket, 'qiniu4x.mp4') ]; -bucketManager.batch(statOperations, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { +bucketManager.batch(statOperations, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { // 200 is success, 298 is part success - if (parseInt(respInfo.statusCode / 100) == 2) { - respBody.forEach(function(item) { - if (item.code == 200) { - console.log(item.data.fsize + "\t" + item.data.hash + "\t" + - item.data.mimeType + "\t" + item.data.putTime + "\t" + + if (parseInt(respInfo.statusCode / 100) == 2) { + respBody.forEach(function (item) { + if (item.code == 200) { + console.log(item.data.fsize + '\t' + item.data.hash + '\t' + + item.data.mimeType + '\t' + item.data.putTime + '\t' + item.data.type); + } else { + console.log(item.code + '\t' + item.data.error); + } + }); } else { - console.log(item.code + "\t" + item.data.error); + console.log(respInfo.statusCode); + console.log(respBody); } - }); - } else { - console.log(respInfo.statusCode); - console.log(respBody); } - } }); diff --git a/examples/rs_bucket_info.js b/examples/rs_bucket_info.js new file mode 100644 index 00000000..3561f828 --- /dev/null +++ b/examples/rs_bucket_info.js @@ -0,0 +1,24 @@ +const qiniu = require('qiniu'); +const proc = require('process'); + +var accessKey = proc.env.QINIU_ACCESS_KEY; +var secretKey = proc.env.QINIU_SECRET_KEY; + +var bucket = proc.env.QINIU_TEST_BUCKET; + +var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); +var config = new qiniu.conf.Config(); +config.zone = qiniu.zone.Zone_z0; +config.useHttpsDomain = 'https'; +var bucketManager = new qiniu.rs.BucketManager(mac, config); +// @param bucketName 空间名 +bucketManager.getBucketInfo(bucket, function (err, respBody, respInfo) { + if (err) { + console.log(err); + throw err; + } + if (respInfo.status == 200) { + console.log('---respBody\n' + JSON.stringify(respBody) + '\n---'); + console.log('---respInfo\n' + JSON.stringify(respInfo) + '\n---'); + } +}); diff --git a/examples/rs_change_mime.js b/examples/rs_change_mime.js index 5be64571..31219785 100644 --- a/examples/rs_change_mime.js +++ b/examples/rs_change_mime.js @@ -1,24 +1,24 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); var bucket = proc.env.QINIU_TEST_BUCKET; var key = 'qiniu.mp4'; var newMime = 'video/x-mp4'; -bucketManager.changeMime(bucket, key, newMime, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - //200 is success - console.log(respInfo.statusCode); - console.log(respBody); - } +bucketManager.changeMime(bucket, key, newMime, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + // 200 is success + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_change_type.js b/examples/rs_change_type.js index 4cd677a2..0807d810 100644 --- a/examples/rs_change_type.js +++ b/examples/rs_change_type.js @@ -1,24 +1,24 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); var bucket = proc.env.QINIU_TEST_BUCKET; var key = 'qiniu.mp4'; -var newType = 1; //低频存储 +var newType = 1; // 低频存储 -bucketManager.changeType(bucket, key, newType, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - //200 is success - console.log(respInfo.statusCode); - console.log(respBody); - } +bucketManager.changeType(bucket, key, newType, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + // 200 is success + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_copy.js b/examples/rs_copy.js index dd5aa4cd..2c972e30 100644 --- a/examples/rs_copy.js +++ b/examples/rs_copy.js @@ -1,30 +1,29 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); -var bucket = proc.env.QINIU_TEST_BUCKET; -var srcKey = "qiniu.mp4"; -var destBucket = srcBucket; -var destKey = "qiniu_new_copy.mp4"; +var srcBucket = proc.env.QINIU_TEST_BUCKET; +var srcKey = 'qiniu.mp4'; +var destBucket = 'destBucket'; +var destKey = 'qiniu_new_copy.mp4'; var options = { - force: true -} - -bucketManager.copy(srcBucket, srcKey, destBucket, destKey, options, function( - err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - //200 is success - console.log(respInfo.statusCode); - console.log(respBody); - } + force: true +}; +bucketManager.copy(srcBucket, srcKey, destBucket, destKey, options, function ( + err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + // 200 is success + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_delete.js b/examples/rs_delete.js index d10c850c..58e72c79 100644 --- a/examples/rs_delete.js +++ b/examples/rs_delete.js @@ -1,22 +1,22 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); var bucket = proc.env.QINIU_TEST_BUCKET; -var key = "qiniu_new_copy.mp4"; +var key = 'qiniu_new_copy.mp4'; -bucketManager.delete(bucket, key, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } +bucketManager.delete(bucket, key, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_delete_after_days.js b/examples/rs_delete_after_days.js index b87ce4a0..97ba55e1 100644 --- a/examples/rs_delete_after_days.js +++ b/examples/rs_delete_after_days.js @@ -1,24 +1,24 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); var bucket = proc.env.QINIU_TEST_BUCKET; -var key = "qiniu_new_copy.mp4"; +var key = 'qiniu_new_copy.mp4'; var days = 10; -bucketManager.deleteAfterDays(bucket, key, days, function(err, respBody, - respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } +bucketManager.deleteAfterDays(bucket, key, days, function (err, respBody, + respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_download.js b/examples/rs_download.js index 010700bd..71d05657 100644 --- a/examples/rs_download.js +++ b/examples/rs_download.js @@ -1,5 +1,5 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; @@ -10,13 +10,13 @@ var publicBucketDomain = 'http://if-pbl.qiniudn.com'; var privateBucketDomain = 'http://if-pri.qiniudn.com'; var key = 'qiniu.mp4'; -//public +// public var publicDownloadUrl = bucketManager.publicDownloadUrl(publicBucketDomain, key); console.log(publicDownloadUrl); -//private -var deadline = parseInt(Date.now() / 1000) + 3600; //1小时过期 +// private +var deadline = parseInt(Date.now() / 1000) + 3600; // 1小时过期 var privateDownloadUrl = bucketManager.privateDownloadUrl(privateBucketDomain, - key, - deadline); + key, + deadline); console.log(privateDownloadUrl); diff --git a/examples/rs_fetch.js b/examples/rs_fetch.js index c072b942..fd6db397 100644 --- a/examples/rs_fetch.js +++ b/examples/rs_fetch.js @@ -1,30 +1,30 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; -//config.zone = qiniu.zone.Zone_z1; +// config.useHttpsDomain = true; +// config.zone = qiniu.zone.Zone_z1; var bucketManager = new qiniu.rs.BucketManager(mac, config); var resUrl = 'http://devtools.qiniu.com/qiniu.png'; var bucket = proc.env.QINIU_TEST_BUCKET; -var key = "qiniu.png"; +var key = 'qiniu.png'; -bucketManager.fetch(resUrl, bucket, key, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - if (respInfo.statusCode == 200) { - console.log(respBody.key); - console.log(respBody.hash); - console.log(respBody.fsize); - console.log(respBody.mimeType); +bucketManager.fetch(resUrl, bucket, key, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; } else { - console.log(respInfo.statusCode); - console.log(respBody); + if (respInfo.statusCode == 200) { + console.log(respBody.key); + console.log(respBody.hash); + console.log(respBody.fsize); + console.log(respBody.mimeType); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } } - } }); diff --git a/examples/rs_list_prefix.js b/examples/rs_list_prefix.js index 682b96a0..ed8d0920 100644 --- a/examples/rs_list_prefix.js +++ b/examples/rs_list_prefix.js @@ -1,11 +1,11 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); @@ -16,35 +16,35 @@ var bucket = proc.env.QINIU_TEST_BUCKET; // limit 每次返回的最大列举文件数量 // delimiter 指定目录分隔符 var options = { - limit: 10, - prefix: 'calculus', + limit: 10, + prefix: 'calculus' }; -bucketManager.listPrefix(bucket, options, function(err, respBody, respInfo) { - if (err) { - console.log(err); - throw err; - } +bucketManager.listPrefix(bucket, options, function (err, respBody, respInfo) { + if (err) { + console.log(err); + throw err; + } - if (respInfo.statusCode == 200) { - //如果这个nextMarker不为空，那么还有未列举完毕的文件列表，下次调用listPrefix的时候， - //指定options里面的marker为这个值 - var nextMarker = respBody.marker; - var commonPrefixes = respBody.commonPrefixes; - console.log(nextMarker); - console.log(commonPrefixes); - var items = respBody.items; - items.forEach(function(item) { - console.log(item.key); - // console.log(item.putTime); - // console.log(item.hash); - // console.log(item.fsize); - // console.log(item.mimeType); - // console.log(item.endUser); - // console.log(item.type); - }); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } + if (respInfo.statusCode == 200) { + // 如果这个nextMarker不为空，那么还有未列举完毕的文件列表，下次调用listPrefix的时候， + // 指定options里面的marker为这个值 + var nextMarker = respBody.marker; + var commonPrefixes = respBody.commonPrefixes; + console.log(nextMarker); + console.log(commonPrefixes); + var items = respBody.items; + items.forEach(function (item) { + console.log(item.key); + // console.log(item.putTime); + // console.log(item.hash); + // console.log(item.fsize); + // console.log(item.mimeType); + // console.log(item.endUser); + // console.log(item.type); + }); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/examples/rs_listv2.js b/examples/rs_listv2.js new file mode 100644 index 00000000..97486e73 --- /dev/null +++ b/examples/rs_listv2.js @@ -0,0 +1,31 @@ +const qiniu = require('qiniu'); +const proc = require('process'); + +var accessKey = proc.env.QINIU_ACCESS_KEY; +var secretKey = proc.env.QINIU_SECRET_KEY; +var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); +var config = new qiniu.conf.Config(); +// config.useHttpsDomain = true; +config.zone = qiniu.zone.Zone_z0; +var bucketManager = new qiniu.rs.BucketManager(mac, config); +var srcBucket = proc.env.QINIU_TEST_BUCKET; +// @param options 列举操作的可选参数 +// prefix 列举的文件前缀 +// marker 上一次列举返回的位置标记，作为本次列举的起点信息 +// limit 每次返回的最大列举文件数量 +// delimiter 指定目录分隔符 +var options = { + limit: 20 +}; + +bucketManager.listPrefixV2(srcBucket, options, function (err, respBody, respInfo) { + // the irregular data return from Server that Cannot be converted by urllib to JSON Object + // so err !=null and you can judge if err.res.statusCode==200 + if (err.res.statusCode != 200) { + console.log(err); + throw err; + } + + console.log('---respBody\n' + respBody + '\n---'); + console.log('---respInfo\n' + JSON.stringify(respInfo) + '\n---'); +}); diff --git a/examples/rs_move.js b/examples/rs_move.js index 5ab446b3..48fed382 100644 --- a/examples/rs_move.js +++ b/examples/rs_move.js @@ -1,28 +1,27 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); var srcBucket = proc.env.QINIU_TEST_BUCKET; -var srcKey = "qiniu.mp4"; +var srcKey = 'qiniu.mp4'; var destBucket = srcBucket; -var destKey = "qiniu_new.mp4"; +var destKey = 'qiniu_new.mp4'; var options = { - force: true -} -bucketManager.move(srcBucket, srcKey, destBucket, destKey, options, function( - err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - //200 is success - console.log(respInfo.statusCode); - } - + force: true +}; +bucketManager.move(srcBucket, srcKey, destBucket, destKey, options, function ( + err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + // 200 is success + console.log(respInfo.statusCode); + } }); diff --git a/examples/rs_prefetch.js b/examples/rs_prefetch.js index d63ddac4..06ae555f 100644 --- a/examples/rs_prefetch.js +++ b/examples/rs_prefetch.js @@ -1,22 +1,22 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z1; var bucketManager = new qiniu.rs.BucketManager(mac, config); var bucket = proc.env.QINIU_TEST_BUCKET; -var key = "qiniu.mp4"; +var key = 'qiniu.mp4'; -bucketManager.prefetch(bucket, key, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - //200 is success - console.log(respInfo.statusCode); - } +bucketManager.prefetch(bucket, key, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; + } else { + // 200 is success + console.log(respInfo.statusCode); + } }); diff --git a/examples/rs_stat.js b/examples/rs_stat.js index 77d22724..b57dcee1 100644 --- a/examples/rs_stat.js +++ b/examples/rs_stat.js @@ -1,31 +1,30 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; var bucketManager = new qiniu.rs.BucketManager(mac, config); var bucket = proc.env.QINIU_TEST_BUCKET; -var key = "qiniux.mp4"; +var key = 'qiniux.mp4'; -bucketManager.stat(bucket, key, function(err, respBody, respInfo) { - if (err) { - console.log(err); - //throw err; - } else { - if (respInfo.statusCode == 200) { - console.log(respBody.hash); - console.log(respBody.fsize); - console.log(respBody.mimeType); - console.log(respBody.putTime); - console.log(respBody.type); +bucketManager.stat(bucket, key, function (err, respBody, respInfo) { + if (err) { + console.log(err); + // throw err; } else { - console.log(respInfo.statusCode); - console.log(respBody.error); + if (respInfo.statusCode == 200) { + console.log(respBody.hash); + console.log(respBody.fsize); + console.log(respBody.mimeType); + console.log(respBody.putTime); + console.log(respBody.type); + } else { + console.log(respInfo.statusCode); + console.log(respBody.error); + } } - } - }); diff --git a/examples/rs_upload_token.js b/examples/rs_upload_token.js index 7eafd4aa..2b92339b 100644 --- a/examples/rs_upload_token.js +++ b/examples/rs_upload_token.js @@ -1,12 +1,12 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var bucket = proc.env.QINIU_TEST_BUCKET; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var putPolicy = new qiniu.rs.PutPolicy({ - scope: bucket + scope: bucket }); var uploadToken = putPolicy.uploadToken(mac); diff --git a/examples/rtc_demo.js b/examples/rtc_demo.js new file mode 100644 index 00000000..6fc6acfb --- /dev/null +++ b/examples/rtc_demo.js @@ -0,0 +1,98 @@ +const qiniu = require('../index.js'); + +// ak, sk 获取参考 https://developer.qiniu.com/dora/kb/3702/QiniuToken +var ACCESS_KEY = 'ak'; +var SECRET_KEY = 'sk'; +var credentials = new qiniu.Credentials(ACCESS_KEY, SECRET_KEY); + +// 参考 https://github.com/pili-engineering/QNRTC-Server/blob/master/docs/api.md + +var data = { + hub: 'your hub', + title: 'your title', + maxUsers: 10, + noAutoKickUser: true +}; + +qiniu.app.createApp(data, credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); + +qiniu.app.getApp('appId', credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); + +qiniu.app.deleteApp('appId', credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); + +var data1 = { + hub: 'your hub', + title: 'your title', + maxUsers: 10, + noAutoKickUser: true, + mergePublishRtmp: { + enable: true, + audioOnly: true, + height: 1920, + width: 1080, + fps: 60, + kbps: 1000, + url: 'rtmp://xxx.example.com/test', + streamTitle: 'meeting' + } +}; +qiniu.app.updateApp('appId', data1, credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); +qiniu.room.listUser('appId', 'roomName', credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); + +qiniu.room.kickUser('appId', 'roomName', 'userId', credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); + +// type of(offset limit) = Num such as 5 10 +qiniu.room.listActiveRooms('appId', 'prefix', 'offset', 'limit', credentials, function (err, res) { + if (err) { + console.log(err); + } else { + console.log(res); + } +}); + +// expireAt = 1524128577 or empty +var roomAccess = { + appId: 'your appId', + roomName: 'your roomName', + userId: 'userId', + expireAt: 1524128577, + permission: 'admin' +}; + +console.log(qiniu.room.getRoomToken(roomAccess, credentials)); diff --git a/examples/sms.js b/examples/sms.js new file mode 100644 index 00000000..14728967 --- /dev/null +++ b/examples/sms.js @@ -0,0 +1,99 @@ +const qiniu = require('../index.js'); +const proc = require('process'); +const should = require('should'); +const assert = require('assert'); + +// eslint-disable-next-line no-undef +before(function(done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + done(); +}); + +// message +describe('test Message', function() { + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + // eslint-disable-next-line no-undef + describe('test sendMessage', function() { + // eslint-disable-next-line no-undef + it('test sendMessage', function(done) { + var num = new Array("17321129884","18120582893"); + var reqBody = { + "template_id": "1199572412090290176", + "mobiles": num, + "parameters": { + "prize": "3333", + "name": "sendMessage", + "time": "1238" + } + }; + qiniu.sms.message.sendMessage(reqBody, mac, function(respErr, respBody, + respInfo) { + should.not.exist(respErr); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + }); + describe('test sendSingleMessage', function() { + it('test sendSingleMessage', function(done) { + var reqBody = { + "template_id": "1199572412090290176", + "mobile": "17321129884", + "parameters": { + "prize": "3333", + "name": "sendSingleMessage", + "time": "1238" + } + }; + qiniu.sms.message.sendSingleMessage(reqBody, mac, function(respErr, respBody, + respInfo) { + should.not.exist(respErr); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + }); + describe('test sendOverseaMessage', function() { + it('test sendOverseaMessage', function(done) { + var reqBody = { + "template_id": "1199572412090290176", + "mobile": "17321129884", + "parameters": { + "prize": "3333", + "name": "1111", + "time": "1238" + } + }; + qiniu.sms.message.sendOverseaMessage(reqBody, mac, function(respErr, respBody, + respInfo) { + should.not.exist(respErr); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + }); + describe('test sendFulltextMessage', function() { + it('test sendFulltextMessage', function(done) { + var num = new Array("17321129884","18120582893"); + var reqBody = { + "mobiles": num, + "content": "【七牛云-测试】您的验证码为1121，该验证码5分钟内有效", + "template_type": "verification" + }; + qiniu.sms.message.sendFulltextMessage(reqBody, mac, function(respErr, respBody, + respInfo) { + if(respErr!=null){ + console.log(respErr); + } + should.not.exist(respErr); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + }); +}); diff --git a/examples/video_pfop.js b/examples/video_pfop.js index 32718859..ee876602 100644 --- a/examples/video_pfop.js +++ b/examples/video_pfop.js @@ -1,43 +1,43 @@ -const qiniu = require("qiniu"); -const proc = require("process"); +const qiniu = require('qiniu'); +const proc = require('process'); var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); -//config.useHttpsDomain = true; +// config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z1; var operManager = new qiniu.fop.OperationManager(mac, config); -//处理指令集合 +// 处理指令集合 var saveBucket = proc.env.QINIU_TEST_BUCKET; var fops = [ - 'avthumb/mp4/s/480x320/vb/150k|saveas/' + qiniu.util.urlsafeBase64Encode( - saveBucket + ":qiniu_480x320.mp4"), - 'vframe/jpg/offset/10|saveas/' + qiniu.util.urlsafeBase64Encode(saveBucket + - ":qiniu_frame1.jpg") + 'avthumb/mp4/s/480x320/vb/150k|saveas/' + qiniu.util.urlsafeBase64Encode( + saveBucket + ':qiniu_480x320.mp4'), + 'vframe/jpg/offset/10|saveas/' + qiniu.util.urlsafeBase64Encode(saveBucket + + ':qiniu_frame1.jpg') ]; var pipeline = 'jemy'; var srcBucket = 'if-bc'; var srcKey = 'qiniu.mp4'; var options = { - 'notifyURL': 'http://api.example.com/pfop/callback', - 'force': false, + notifyURL: 'http://api.example.com/pfop/callback', + force: false }; -//持久化数据处理返回的是任务的persistentId，可以根据这个id查询处理状态 -operManager.pfop(srcBucket, srcKey, fops, pipeline, options, function(err, - respBody, - respInfo) { - if (err) { - throw err; - } +// 持久化数据处理返回的是任务的persistentId，可以根据这个id查询处理状态 +operManager.pfop(srcBucket, srcKey, fops, pipeline, options, function (err, + respBody, + respInfo) { + if (err) { + throw err; + } - if (respInfo.statusCode == 200) { - console.log(respBody.persistentId); - } else { - console.log(respInfo.statusCode); - console.log(respBody); - } + if (respInfo.statusCode == 200) { + console.log(respBody.persistentId); + } else { + console.log(respInfo.statusCode); + console.log(respBody); + } }); diff --git a/index.d.ts b/index.d.ts index 92c9f396..db6a7a72 100644 --- a/index.d.ts +++ b/index.d.ts @@ -3,15 +3,24 @@ * @date 2017-06-27 * @author xialeistudio */ +import { Callback, RequestOptions } from 'urllib'; +import { Agent as HttpAgent, IncomingMessage} from 'http'; +import { Agent as HttpsAgent } from 'https'; +import { Readable } from "stream"; + export declare type callback = (e?: Error, respBody?: any, respInfo?: any) => void; export declare namespace auth { namespace digest { + interface MacOptions { + disableQiniuTimestampSignature?: boolean; + } + class Mac { accessKey: string; secretKey: string; - constructor(accessKey?: string, secretKey?: string); + constructor(accessKey?: string, secretKey?: string, options?: MacOptions); } } } @@ -128,12 +137,17 @@ export declare namespace conf { /** * @default null */ - zone?: 'huadong' | 'huabei' | 'huanan' | 'beimei'; + zone?: Zone, /** * @default -1 */ zoneExpire?: number; + + /** + * @default null + */ + regionsProvider?: httpc.RegionsProvider; } class Config implements ConfigOptions { constructor(options?: ConfigOptions); @@ -152,6 +166,11 @@ export declare namespace conf { } export declare namespace form_up { + type UploadResult = { + data: any; + resp: IncomingMessage; + } + class FormUploader { conf: conf.Config; @@ -161,11 +180,17 @@ export declare namespace form_up { * * @param uploadToken * @param key - * @param rsStream + * @param fsStream * @param putExtra * @param callback */ - putStream(uploadToken: string, key: string | null, rsStream: NodeJS.ReadableStream, putExtra: PutExtra | null, callback: callback): void; + putStream( + uploadToken: string, + key: string | null, + fsStream: NodeJS.ReadableStream, + putExtra: PutExtra | null, + callback: callback + ): Promise; /** * @@ -175,7 +200,13 @@ export declare namespace form_up { * @param putExtra * @param callback */ - put(uploadToken: string, key: string | null, body: any, putExtra: PutExtra | null, callback: callback): void; + put( + uploadToken: string, + key: string | null, + body: any, + putExtra: PutExtra | null, + callback: callback + ): Promise; /** * @@ -184,7 +215,12 @@ export declare namespace form_up { * @param putExtra * @param callback */ - putWithoutKey(uploadToken: string, body: any, putExtra: PutExtra | null, callback: callback): void; + putWithoutKey( + uploadToken: string, + body: any, + putExtra: PutExtra | null, + callback: callback + ): Promise; /** * 上传本地文件 @@ -194,7 +230,13 @@ export declare namespace form_up { * @param putExtra 额外选项 * @param callback */ - putFile(uploadToken: string, key: string | null, localFile: string, putExtra: PutExtra | null, callback: callback): void; + putFile( + uploadToken: string, + key: string | null, + localFile: string, + putExtra: PutExtra | null, + callback: callback + ): Promise; /** * @@ -203,7 +245,12 @@ export declare namespace form_up { * @param putExtra * @param callback */ - putFileWithoutKey(uploadToken: string, localFile: string, putExtra: PutExtra | null, callback: callback): void; + putFileWithoutKey( + uploadToken: string, + localFile: string, + putExtra: PutExtra | null, + callback: callback + ): Promise; } class PutExtra { @@ -215,7 +262,7 @@ export declare namespace form_up { /** * @default {} */ - params: any; + params: Record; /** * @default null @@ -232,6 +279,11 @@ export declare namespace form_up { */ checkCrc?: number | boolean; + /** + * @default {} + */ + metadata?: Record; + /** * 上传可选参数 * @param fname 请求体中的文件的名称 @@ -239,12 +291,18 @@ export declare namespace form_up { * @param mimeType 指定文件的mimeType * @param crc32 指定文件的crc32值 * @param checkCrc 指定是否检测文件的crc32值 + * @param metadata 元数据设置，参数名称必须以 x-qn-meta-${name}: 开头 */ - constructor(fname?: string, params?: any, mimeType?: string, crc32?: string, checkCrc?: number | boolean); + constructor(fname?: string, params?: Record, mimeType?: string, crc32?: string, checkCrc?: number | boolean, metadata?: Record); } } export declare namespace resume_up { + type UploadResult = { + data: any; + resp: IncomingMessage; + } + class ResumeUploader { config: conf.Config; @@ -259,7 +317,14 @@ export declare namespace resume_up { * @param putExtra * @param callback */ - putStream(uploadToken: string, key: string | null, rsStream: NodeJS.ReadableStream, rsStreamLen: number, putExtra: PutExtra | null, callback: callback): void; + putStream( + uploadToken: string, + key: string | null, + rsStream: NodeJS.ReadableStream, + rsStreamLen: number, + putExtra: PutExtra | null, + callback: callback + ): Promise; /** * @@ -269,7 +334,13 @@ export declare namespace resume_up { * @param putExtra * @param callback */ - putFile(uploadToken: string, key: string | null, localFile: string, putExtra: PutExtra | null, callback: callback): void; + putFile( + uploadToken: string, + key: string | null, + localFile: string, + putExtra: PutExtra | null, + callback: callback + ): Promise; /** * @@ -278,25 +349,55 @@ export declare namespace resume_up { * @param putExtra * @param callback */ - putFileWithoutKey(uploadToken: string, localFile: string, putExtra: PutExtra | null, callback: callback): void; + putFileWithoutKey( + uploadToken: string, + localFile: string, + putExtra: PutExtra | null, + callback: callback + ): Promise; } class PutExtra { /** * @default '' */ - fname: string; + fname?: string; /** * @default {} */ - params: any; + params?: Record; /** * @default null */ mimeType?: string; + /** + * @default null + */ + resumeRecordFile?: string + + /** + * @default null + */ + progressCallback?: (uploadBytes: number, totalBytes: number) => void + + /** + * @default v1 + */ + version?: string + + /** + * @default 4 * 1024 * 1024 + */ + partSize?: number + + /** + * @default {} + */ + metadata?: Record + /** * 上传可选参数 * @param fname 请求体中的文件的名称 @@ -304,14 +405,34 @@ export declare namespace resume_up { * @param mimeType 指定文件的mimeType * @param resumeRecordFile * @param progressCallback + * @param partSize 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + * @param version 分片上传版本 目前支持v1/v2版本 默认v1 + * @param metadata 元数据设置，参数名称必须以 x-qn-meta-${name}: 开头 */ - constructor(fname?: string, params?: any, mimeType?: string, resumeRecordFile?: string, progressCallback?: (data: any) => void); + constructor(fname?: string, params?: Record, mimeType?: string, resumeRecordFile?: string, + progressCallback?: (uploadBytes: number, totalBytes: number) => void, + partSize?:number, version?:string, metadata?: Record); } } export declare namespace util { function isTimestampExpired(timestamp: number): boolean; + /** + * 使用 UTC 时间来格式化日期时间 + * + * @param date 与 new Date() 接受的参数一样，内部会使用 new Date(date) 生成日期时间对象 + * @param layout 目前仅接受 + * YYYY + * MM + * DD + * HH + * mm + * ss + * SSS + */ + function formatDateUTC(date: Date | number | string, layout?: string): string; + function encodedEntry(bucket: string, key?: string): string; function getAKFromUptoken(uploadToken: string): string; @@ -328,6 +449,8 @@ export declare namespace util { function hmacSha1(encodedFlags: string | Buffer, secretKey: string | Buffer): string; + function canonicalMimeHeaderKey(fieldName: string): string; + /** * 创建AccessToken凭证 * @param mac AK&SK对象 @@ -344,8 +467,9 @@ export declare namespace util { * @param reqMethod 请求方法，例如 GET，POST * @param reqContentType 请求类型，例如 application/json 或者 application/x-www-form-urlencoded * @param reqBody 请求Body，仅当请求的 ContentType 为 application/json 或者 application/x-www-form-urlencoded 时才需要传入该参数 + * @param reqHeaders 请求Headers，例如 {"X-Qiniu-Name": "Qiniu", "Content-Type": "application/x-www-form-urlencoded"} */ - function generateAccessTokenV2(mac: auth.digest.Mac, requestURI: string, reqMethod: string, reqContentType: string, reqBody?: string): string; + function generateAccessTokenV2(mac: auth.digest.Mac, requestURI: string, reqMethod: string, reqContentType: string, reqBody?: string, reqHeaders?: Record): string; /** * 校验七牛上传回调的Authorization @@ -357,11 +481,380 @@ export declare namespace util { function isQiniuCallback(mac: auth.digest.Mac, requestURI: string, reqBody: string | null, callbackAuth: string): boolean; } +export declare namespace httpc { + interface ReqOpts { + agent?: HttpAgent; + httpsAgent?: HttpsAgent; + url: string; + middlewares: middleware.Middleware[]; + callback?: Callback; + urllibOptions: RequestOptions; + } + + interface ResponseWrapperOptions { + data: T; + resp: IncomingMessage; + } + + // responseWrapper.js + class ResponseWrapper { + data: T; + resp: IncomingMessage; + constructor(options: ResponseWrapperOptions); + ok(): boolean; + needRetry(): boolean; + } + + // middleware package + namespace middleware { + interface Middleware { + send( + request: ReqOpts, + next: (reqOpts: ReqOpts) => Promise> + ): Promise>; + } + + /** + * 组合中间件为一个调用函数 + * @param middlewares 中间件列表 + * @param handler 请求函数 + */ + function composeMiddlewares( + middlewares: Middleware[], + handler: (reqOpts: ReqOpts) => Promise> + ); + + /** + * 设置 User-Agent 请求头中间件 + */ + class UserAgentMiddleware implements Middleware { + constructor(sdkVersion: string); + send( + request: httpc.ReqOpts, + next: (reqOpts: httpc.ReqOpts) => Promise> + ): Promise>; + } + + interface RetryDomainsMiddlewareOptions { + backupDomains: string[]; + maxRetryTimes: number; + retryCondition: () => boolean; + } + + class RetryDomainsMiddleware implements Middleware { + /** + * 备用域名 + */ + backupDomains: string[]; + + /** + * 最大重试次数，包括首次请求 + */ + maxRetryTimes: number; + + /** + * 是否可以重试，可以通过该函数配置更详细的重试规则 + */ + retryCondition: () => boolean; + + /** + * 已经重试的次数 + * @private + */ + private _retriedTimes: number; + + /** + * 实例化重试域名中间件 + * @param retryDomainsOptions + */ + constructor(retryDomainsOptions: RetryDomainsMiddlewareOptions) + + /** + * 重试域名中间件逻辑 + * @param request + * @param next + */ + send( + request: httpc.ReqOpts, + next: (reqOpts: httpc.ReqOpts) => Promise> + ): Promise>; + + /** + * 控制重试逻辑，主要为 {@link retryCondition} 服务。若没有设置 retryCondition，默认 2xx 才会终止重试 + * @param err + * @param respWrapper + * @param reqOpts + * @private + */ + private _shouldRetry( + err: Error | null, + respWrapper: ResponseWrapper, + reqOpts: ReqOpts + ): boolean; + } + } + + // client.js + interface HttpClientOptions { + httpAgent?: HttpAgent; + httpsAgent?: HttpsAgent; + middlewares?: middleware.Middleware[]; + } + + interface GetOptions extends ReqOpts { + params: Record; + headers: Record; + } + + interface PostOptions extends ReqOpts { + data: string | Buffer | Readable; + headers: Record; + } + + interface PutOptions extends ReqOpts { + data: string | Buffer | Readable; + headers: Record + } + + class HttpClient { + httpAgent: HttpAgent; + httpsAgent: HttpsAgent; + middlewares: middleware.Middleware[]; + constructor(options: HttpClientOptions) + sendRequest(requestOptions: ReqOpts): Promise + get(getOptions: GetOptions): Promise + post(postOptions: PostOptions): Promise + put(putOptions: PutOptions): Promise + } + + // endpoint.js + interface EndpointOptions { + defaultScheme?: string; + } + + interface EndpointPersistInfo { + host: string; + defaultScheme: string; + } + + class Endpoint { + static fromPersistInfo(persistInfo: EndpointPersistInfo): Endpoint; + + host: string; + defaultScheme: string; + + constructor(host: string, options?: EndpointOptions); + + getValue(options?: {scheme?: string}): string; + + get persistInfo(): EndpointPersistInfo; + } + + // region.js + enum SERVICE_NAME { + UC = 'uc', + UP = 'up', + IO = 'io', + RS = 'rs', + RSF = 'rsf', + API = 'api', + S3 = 's3' + } + + interface RegionOptions { + regionId?: string; + s3RegionId?: string; + services?: Record; + ttl?: number; + createTime?: Date; + } + + interface RegionFromZoneOptions { + regionId?: string; + s3RegionId?: string; + ttl?: number; + isPreferCdnHost?: boolean; + } + + interface RegionFromRegionIdOptions { + s3RegionId?: string; + ttl?: number; + createTime?: Date; + extendedServices?: Record + } + + interface RegionPersistInfo { + regionId?: string; + s3RegionId?: string; + services: Record; + ttl: number; + createTime: number; + } + + interface QueryRegionsRespData { + region: string; + ttl: number; + s3: { + domains: string[]; + region_alias: string; + }; + uc: { + domains: string[]; + }; + up: { + domains: string[]; + }; + io: { + domains: string[]; + }; + rs: { + domains: string[]; + }; + rsf: { + domains: string[]; + }; + api: { + domains: string[]; + }; + } + + class Region { + static fromZone(zone: conf.Zone, options?: RegionFromZoneOptions): Region; + static fromRegionId(regionId: string, options?: RegionFromRegionIdOptions): Region; + static fromPersistInfo(persistInfo: RegionPersistInfo): Region; + static fromQueryData(data: QueryRegionsRespData): Region; + + // non-unique + regionId?: string; + s3RegionId?: string; + services: Record + + ttl: number; + createTime: Date; + + constructor(options: RegionOptions); + + get isLive(): boolean; + get persistInfo(): RegionPersistInfo; + } + + // endpointProvider.js + interface EndpointsProvider { + getEndpoints(): Promise + } + + interface MutableEndpointsProvider extends EndpointsProvider { + setEndpoints(endpoints: Endpoint[]): Promise + } + + class StaticEndpointsProvider implements EndpointsProvider { + static fromRegion(region: Region, serviceName: SERVICE_NAME | string): StaticEndpointsProvider; + + constructor(endpoints: Endpoint[]); + + getEndpoints(): Promise; + } + + // regionsProvider.js + interface RegionsProvider { + getRegions(): Promise + } + + interface MutableRegionsProvider extends RegionsProvider { + setRegions(regions: Region[]): Promise + } + + // StaticRegionsProvider + class StaticRegionsProvider implements RegionsProvider { + regions: Region[]; + + constructor(regions: Region[]); + + getRegions(): Promise; + } + + // CachedRegionsProviderOptions + interface CachedRegionsProviderOptions { + cacheKey: string; + baseRegionsProvider: RegionsProvider; + persistPath?: string; + shrinkInterval?: number; // ms + } + + class CachedRegionsProvider implements MutableRegionsProvider { + cacheKey: string; + baseRegionsProvider: RegionsProvider; + + lastShrinkAt: Date; + shrinkInterval: number; + + constructor( + options: CachedRegionsProviderOptions + ); + + setRegions(regions: Region[]): Promise; + + getRegions(): Promise; + } + + // QueryRegionsProvider + interface QueryRegionsProviderOptions { + accessKey: string; + bucketName: string; + endpointsProvider: EndpointsProvider; + } + + class QueryRegionsProvider implements RegionsProvider { + accessKey: string; + bucketName: string; + endpointsProvider: EndpointsProvider; + + constructor(options: QueryRegionsProviderOptions); + + getRegions(): Promise; + } +} + export declare namespace rpc { - interface Headers { + type Headers = Record & { 'User-Agent'?: string; Connection?: string; } + + interface RequestOptions { + headers: Headers; + mac: auth.digest.Mac; + } + + const qnHttpClient: httpc.HttpClient; + + /** + * + * @param requestUrl 请求地址 + * @param headers 请求 headers + * @param callbackFunc 回调函数 + */ + function get(requestUrl: string, headers: Headers | null, callbackFunc: callback): void; + + /** + * @param requestUrl 请求地址 + * @param options 请求的配置 + * @param callbackFunc 回调函数 + */ + function getWithOptions( + requestUrl: string, + options: RequestOptions | null, + callbackFunc: callback + ): ReturnType; + + /** + * + * @param requestUrl 请求地址 + * @param token 请求认证签名 + * @param callbackFunc 回调函数 + */ + function getWithToken(requestUrl: string, token: string | null, callbackFunc: callback): void; + /** * * @param requestURI @@ -371,6 +864,20 @@ export declare namespace rpc { */ function post(requestURI: string, requestForm: Buffer | string | NodeJS.ReadableStream | null, headers: Headers | null, callback: callback): void; + + /** + * @param requestUrl 请求地址 + * @param requestForm 请求体 + * @param options 请求的配置 + * @param callbackFunc 回调函数 + */ + function postWithOptions( + requestUrl: string, + requestForm: Buffer | string | NodeJS.ReadableStream | null, + options: RequestOptions | null, + callbackFunc: callback + ): ReturnType; + /** * * @param requestURI @@ -400,12 +907,16 @@ export declare namespace rpc { export declare namespace zone { //huadong const Zone_z0: conf.Zone; + //huadong2 + const Zone_cn_east_2: conf.Zone; //huabei const Zone_z1: conf.Zone; //huanan const Zone_z2: conf.Zone; //beimei const Zone_na0: conf.Zone; + //Southeast Asia + const Zone_as0: conf.Zone; } export declare namespace fop { @@ -469,6 +980,18 @@ export declare namespace rs { delimiter?: string; } + type BucketEventName = 'put' + | 'mkfile' + | 'delete' + | 'copy' + | 'move' + | 'append' + | 'disable' + | 'enable' + | 'deleteMarkerCreate' + | 'predelete' + | 'restore:completed'; + class BucketManager { mac: auth.digest.Mac; config: conf.Config; @@ -544,7 +1067,7 @@ export declare namespace rs { delete(bucket: string, key: string, callback: callback): void; /** - * 更新文件的生命周期 + * 设置文件删除的生命周期 * @see https://developer.qiniu.com/kodo/api/1732/update-file-lifecycle * * @param bucket 空间名称 @@ -554,6 +1077,50 @@ export declare namespace rs { */ deleteAfterDays(bucket: string, key: string, days: number, callback: callback): void; + /** + * 设置文件的生命周期 + * @param { string } bucket - 空间名称 + * @param { string } key - 文件名称 + * @param { Object } options - 配置项 + * @param { number } options.toIaAfterDays - 多少天后将文件转为低频存储，设置为 -1 表示取消已设置的转低频存储的生命周期规则， 0 表示不修改转低频生命周期规则。 + * @param { number } options.toArchiveIRAfterDays - 多少天后将文件转为归档直读存储，设置为 -1 表示取消已设置的转归档直读存储的生命周期规则， 0 表示不修改转归档直读生命周期规则。 + * @param { number } options.toArchiveAfterDays - 多少天后将文件转为归档存储，设置为 -1 表示取消已设置的转归档存储的生命周期规则， 0 表示不修改转归档生命周期规则。 + * @param { number } options.toDeepArchiveAfterDays - 多少天后将文件转为深度归档存储，设置为 -1 表示取消已设置的转深度归档存储的生命周期规则， 0 表示不修改转深度归档生命周期规则。 + * @param { number } options.deleteAfterDays - 多少天后将文件删除，设置为 -1 表示取消已设置的删除存储的生命周期规则， 0 表示不修改删除存储的生命周期规则。 + * @param { Object } options.cond - 匹配条件，只有条件匹配才会设置成功 + * @param { string } options.cond.hash + * @param { string } options.cond.mime + * @param { number } options.cond.fsize + * @param { number } options.cond.putTime + * @param { function } callbackFunc - 回调函数 + */ + setObjectLifeCycle( + bucket: string, + key: string, + options: { + toIaAfterDays?: number, + toArchiveIRAfterDays?: number, + toArchiveAfterDays?: number, + toDeepArchiveAfterDays?: number, + deleteAfterDays?: number + cond?: { + hash?: string, + mime?: string, + fsize?: number, + putTime?: number + } + }, + callbackFunc: callback + ): void; + + /** + * 解冻归档存储文件 + * @param entry + * @param freezeAfterDays + * @param callbackFunc + */ + restoreAr(entry: string, freezeAfterDays: number, callbackFunc: callback): void; + /** * 抓取资源 * @see https://developer.qiniu.com/kodo/api/1263/fetch @@ -612,10 +1179,21 @@ export declare namespace rs { * * @param bucket 空间名称 * @param options 列举操作的可选参数 - * @param callback + * @param callback 回调函数 */ listPrefix(bucket: string, options: ListPrefixOptions | null, callback: callback): void; + /** + * 获取制定前缀的文件列表 V2 + * + * @deprecated API 可能返回仅包含 marker，不包含 item 或 dir 的项，请使用 {@link listPrefix} + * + * @param bucket 空间名称 + * @param options 列举操作的可选参数 + * @param callback 回调函数 + */ + listPrefixV2(bucket: string, options: ListPrefixOptions | null, callback: callback): void; + /** * 批量文件管理请求，支持stat，chgm，chtype，delete，copy，move * @param operations @@ -637,6 +1215,195 @@ export declare namespace rs { * @param fileName 原始文件名 */ publicDownloadUrl(domain: string, fileName: string): string; + + /** + * rules/add 增加 bucket 规则 + * + * @param bucket - 空间名 + * + * @param options - 配置项 + * @param options.name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param options.prefix - 同一个 bucket 里面前缀不能重复 + * @param options.to_line_after_days - 指定文件上传多少天后转低频存储。指定为0表示不转低频存储 + * @param options.to_archive_ir_after_days - 指定文件上传多少天后转归档直读存储。指定为0表示不转归档直读存储 + * @param options.to_archive_after_days - 指定文件上传多少天后转归档存储。指定为0表示不转归档存储 + * @param options.to_deep_archive_after_days - 指定文件上传多少天后转深度归档存储。指定为0表示不转深度归档存储 + * @param options.delete_after_days - 指定上传文件多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param options.history_delete_after_days - 指定文件成为历史版本多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param options.history_to_line_after_days - 指定文件成为历史版本多少天后转低频存储。指定为0表示不转低频存储 + * + * @param callbackFunc - 回调函数 + */ + putBucketLifecycleRule( + bucket: string, + options: { + name: string, + prefix?: string, + to_line_after_days?: number, + to_archive_ir_after_days?: number, + to_archive_after_days?: number, + to_deep_archive_after_days?: number, + delete_after_days?: number, + history_delete_after_days?: number, + history_to_line_after_days?: number, + }, + callbackFunc: callback + ): void; + + /** rules/delete 删除 bucket 规则 + * @param bucket - 空间名 + * @param name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param callbackFunc - 回调函数 + */ + deleteBucketLifecycleRule(bucket: string, name: string, callbackFunc: callback): void; + + /** + * rules/update 更新 bucket 规则 + * + * @param bucket - 空间名 + * + * @param options - 配置项 + * @param options.name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param options.prefix - 同一个 bucket 里面前缀不能重复 + * @param options.to_line_after_days - 指定文件上传多少天后转低频存储。指定为0表示不转低频存储 + * @param options.to_archive_ir_after_days - 指定文件上传多少天后转归档直读存储。指定为0表示不转归档直读存储 + * @param options.to_archive_after_days - 指定文件上传多少天后转归档存储。指定为0表示不转归档存储 + * @param options.to_deep_archive_after_days - 指定文件上传多少天后转深度归档存储。指定为0表示不转深度归档存储 + * @param options.delete_after_days - 指定上传文件多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param options.history_delete_after_days - 指定文件成为历史版本多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param options.history_to_line_after_days - 指定文件成为历史版本多少天后转低频存储。指定为0表示不转低频存储 + * + * @param callbackFunc - 回调函数 + */ + updateBucketLifecycleRule( + bucket: string, + options: { + name: string, + prefix?: string, + to_line_after_days?: number, + to_archive_ir_after_days?: number, + to_archive_after_days?: number, + to_deep_archive_after_days?: number, + delete_after_days?: number, + history_delete_after_days?: number, + history_to_line_after_days?: number, + }, + callbackFunc: callback + ): void; + + + /** rules/get - 获取 bucket 规则 + * @param bucket - 空间名 + * @param callbackFunc - 回调函数 + */ + getBucketLifecycleRule(bucket: string, callbackFunc: callback): void + + /** + * 添加事件通知 + * https://developer.qiniu.com/kodo/8610/dev-event-notification + * @param bucket - 空间名 + * @param options - 配置项 + * @param options.name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param options.event - 事件类型，接受数组设置多个 + * @param options.callbackUrl - 事件通知回调 URL，接受数组设置多个，失败依次重试 + * @param options.prefix - 可选，文件配置的前缀 + * @param options.suffix - 可选，文件配置的后缀 + * @param options.access_key - 可选，设置的话会对通知请求用对应的ak、sk进行签名 + * @param options.host - 可选，通知请求的host + * @param callbackFunc - 回调函数 + */ + putBucketEvent( + bucket: string, + options: { + name: string, + event: BucketEventName | BucketEventName[], + callbackUrl: string | string[], + prefix?: string, + suffix?: string, + access_key?: string, + host?: string, + }, + callbackFunc: callback, + ): void + + /** + * 更新事件通知 + * https://developer.qiniu.com/kodo/8610/dev-event-notification + * @param bucket - 空间名 + * @param options - 配置项 + * @param options.name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param options.event - 事件类型，接受数组设置多个 + * @param options.callbackUrl - 事件通知回调 URL，接受数组设置多个，失败依次重试 + * @param options.prefix - 可选，文件配置的前缀 + * @param options.suffix - 可选，文件配置的后缀 + * @param options.access_key - 可选，设置的话会对通知请求用对应的ak、sk进行签名 + * @param options.host - 可选，通知请求的host + * @param callbackFunc - 回调函数 + */ + updateBucketEvent( + bucket: string, + options: { + name: string, + event?: BucketEventName | BucketEventName[], + callbackUrl?: string | string[], + prefix?: string, + suffix?: string, + access_key?: string, + host?: string, + }, + callbackFunc: callback, + ): void + + /** + * 获取事件通知规则 + * https://developer.qiniu.com/kodo/8610/dev-event-notification + * + * @param bucket - 空间名 + * @param callbackFunc - 回调函数 + */ + getBucketEvent(bucket: string, callbackFunc: callback): void + + /** + * 删除事件通知规则 + * https://developer.qiniu.com/kodo/8610/dev-event-notification + * + * @param bucket - 空间名 + * @param name - 规则名称 + * @param callbackFunc - 回调函数 + */ + deleteBucketEvent(bucket: string, name: string, callbackFunc: callback): void + + /** + * 设置 bucket 的 cors（跨域）规则 + * https://developer.qiniu.com/kodo/8539/set-the-cross-domain-resource-sharing + * @param bucket - 空间名 + * @param body - 规则配置 + * @param body[].allowed_origin - 允许的域名 + * @param body[].allowed_method - 允许的请求方法；大小写不敏感 + * @param body[].allowed_header - 可选，允许的 header；默认不允许任何 header；大小写不敏感 + * @param body[].exposed_header - 可选，暴露的 header；默认 X-Log, X-Reqid；大小写不敏感 + * @param body[].max_age - 可选，结果可以缓存的时间；默认不缓存 + * @param callbackFunc - 回调函数 + */ + putCorsRules( + bucket: string, + body: { + allowed_origin: string[], + allowed_method: string[], + allowed_header?: string[], + exposed_header?: string[], + max_age?: number, + }[], + callbackFunc: callback + ): void + + /** + * 获取 bucket 的 cors（跨域）规则 + * https://developer.qiniu.com/kodo/8539/set-the-cross-domain-resource-sharing + * @param bucket - 空间名 + * @param callbackFunc - 回调函数 + */ + getCorsRules(bucket: string, callbackFunc: callback): void } /** @@ -711,6 +1478,7 @@ export declare namespace rs { expires?: number; insertOnly?: number; saveKey?: string; + forceSaveKey?: boolean; endUser?: string; returnUrl?: string; returnBody?: string; @@ -731,8 +1499,19 @@ export declare namespace rs { detectMime?: number; deleteAfterDays?: number; fileType?: number; + + // @deprecated + transform?: string; + // @deprecated + transformFallbackMode?: string; + // @deprecated + transformFallbackKey?: string; + + [key: string]: string | number | boolean; } class PutPolicy { + [k: string]: string | number | boolean | Function; + constructor(options?: PutPolicyOptions); getFlags(): any; @@ -740,3 +1519,75 @@ export declare namespace rs { uploadToken(mac?: auth.digest.Mac): string; } } + +export declare namespace sms { + namespace message { + /** + * 发送短信 (POST Message) + * @link https://developer.qiniu.com/sms/5897/sms-api-send-message#1 + * @param reqBody + * @param mac + * @param callback + */ + function sendMessage( + reqBody: { + "template_id": string, + "mobiles": string[], + "parameters"?: Record + }, + mac: auth.digest.Mac, + callback: Callback<{ job_id: string }> + ): void; + + /** + * 发送单条短信 (POST Single Message) + * @link https://developer.qiniu.com/sms/5897/sms-api-send-message#2 + * @param reqBody + * @param mac + * @param callback + */ + function sendSingleMessage( + reqBody: { + "template_id": string, + "mobile": string, + "parameters"?: Record + }, + mac: auth.digest.Mac, + callback: Callback<{ message_id: string }> + ): void; + + /** + * 发送国际/港澳台短信 (POST Oversea Message) + * @link https://developer.qiniu.com/sms/5897/sms-api-send-message#3 + * @param reqBody + * @param mac + * @param callback + */ + function sendOverseaMessage( + reqBody: { + "template_id": string, + "mobile": string, + "parameters"?: Record + }, + mac: auth.digest.Mac, + callback: Callback<{ message_id: string }> + ): void; + + /** + * 发送全文本短信(不需要传模版 ID) (POST Fulltext Message) + * @link https://developer.qiniu.com/sms/5897/sms-api-send-message#4 + * @param reqBody + * @param mac + * @param callback + */ + function sendFulltextMessage( + reqBody: { + "mobiles": string[], + "content": string, + "template_type": string + }, + mac: auth.digest.Mac, + callback: Callback<{ job_id: string }> + ): void; + } +} diff --git a/index.js b/index.js index db6efa33..315fc6ea 100644 --- a/index.js +++ b/index.js @@ -1,16 +1,33 @@ -var libPath = process.env.QINIU_COV ? './lib-cov' : './qiniu'; - module.exports = { - auth: { - digest: require(libPath + '/auth' + '/digest.js') - }, - cdn: require(libPath + "/cdn.js"), - form_up: require(libPath + '/storage/form.js'), - resume_up: require(libPath + '/storage/resume.js'), - rs: require(libPath + '/storage/rs.js'), - fop: require(libPath + '/fop.js'), - conf: require(libPath + '/conf.js'), - rpc: require(libPath + '/rpc.js'), - util: require(libPath + '/util.js'), - zone: require(libPath + '/zone.js') + auth: { + digest: require('./qiniu/auth/digest.js') + }, + cdn: require('./qiniu/cdn.js'), + form_up: require('./qiniu/storage/form.js'), + resume_up: require('./qiniu/storage/resume.js'), + rs: require('./qiniu/storage/rs.js'), + fop: require('./qiniu/fop.js'), + conf: require('./qiniu/conf.js'), + httpc: { + middleware: require('./qiniu/httpc/middleware'), + HttpClient: require('./qiniu/httpc/client').HttpClient, + ResponseWrapper: require('./qiniu/httpc/responseWrapper').ResponseWrapper, + Endpoint: require('./qiniu/httpc/endpoint').Endpoint, + StaticEndpointsProvider: require('./qiniu/httpc/endpointsProvider').StaticEndpointsProvider, + SERVICE_NAME: require('./qiniu/httpc/region').SERVICE_NAME, + Region: require('./qiniu/httpc/region').Region, + StaticRegionsProvider: require('./qiniu/httpc/regionsProvider').StaticRegionsProvider, + CachedRegionsProvider: require('./qiniu/httpc/regionsProvider').CachedRegionsProvider, + QueryRegionsProvider: require('./qiniu/httpc/regionsProvider').QueryRegionsProvider, + ChainedRegionsProvider: require('./qiniu/httpc/regionsProvider').ChainedRegionsProvider + }, + rpc: require('./qiniu/rpc.js'), + util: require('./qiniu/util.js'), + zone: require('./qiniu/zone.js'), + app: require('./qiniu/rtc/app.js'), + room: require('./qiniu/rtc/room.js'), + Credentials: require('./qiniu/rtc/credentials.js'), + sms: { + message: require('./qiniu/sms/message.js') + } }; diff --git a/package.json b/package.json index a783d1c5..252f4866 100644 --- a/package.json +++ b/package.json @@ -1,22 +1,16 @@ { "name": "qiniu", - "version": "7.1.3", + "version": "7.11.1", "description": "Node wrapper for Qiniu Resource (Cloud) Storage API", "main": "index.js", "directories": { "test": "test" }, "scripts": { - "test": "make test", - "blanket": { - "pattern": "qiniu/qiniu", - "data-cover-flags": { - "debug": false - } - }, - "travis-cov": { - "threshold": 90 - } + "test": "NODE_ENV=test mocha -t 300000", + "cover": "nyc npm run test", + "report": "nyc report --reporter=html", + "lint": "eslint ." }, "repository": { "type": "git", @@ -51,25 +45,33 @@ "email": "jinxinxin@qiniu.com" } ], - "engines": [ - "node >= 4" - ], + "engines": { + "node": ">= 6" + }, "dependencies": { - "agentkeepalive": "3.3.0", - "crc32": "0.2.2", + "agentkeepalive": "^4.0.2", + "before": "^0.0.1", + "block-stream2": "^2.0.0", + "crc32": "^0.2.2", + "destroy": "^1.0.4", "encodeurl": "^1.0.1", - "formstream": "1.1.0", - "mime": "1.3.6", - "tunnel-agent": "0.6.0", - "urllib": "2.22.0" + "formstream": "^1.1.0", + "mime": "^2.4.4", + "mockdate": "^3.0.5", + "tunnel-agent": "^0.6.0", + "urllib": "^2.41.0" }, "devDependencies": { "@types/node": "^8.0.3", - "blanket": "*", - "mocha": "*", - "pedding": "*", - "should": "1.2.2", - "travis-cov": "*" + "eslint": "^6.5.1", + "eslint-config-standard": "^14.1.0", + "eslint-plugin-import": "^2.11.0", + "eslint-plugin-node": "^10.0.0", + "eslint-plugin-promise": "^4.2.1", + "eslint-plugin-standard": "^4.0.1", + "mocha": "^6.2.1", + "nyc": "^14.1.1", + "should": "^13.2.3" }, "license": "MIT" } diff --git a/qiniu/auth/digest.js b/qiniu/auth/digest.js index 669aadd0..521ef5ce 100644 --- a/qiniu/auth/digest.js +++ b/qiniu/auth/digest.js @@ -1,10 +1,13 @@ -var url = require('url'); var conf = require('../conf'); -var util = require('../util'); exports.Mac = Mac; -function Mac(accessKey, secretKey) { - this.accessKey = accessKey || conf.ACCESS_KEY; - this.secretKey = secretKey || conf.SECRET_KEY; +const defaultOptions = { + disableQiniuTimestampSignature: null +}; + +function Mac (accessKey, secretKey, options) { + this.accessKey = accessKey || conf.ACCESS_KEY; + this.secretKey = secretKey || conf.SECRET_KEY; + this.options = Object.assign({}, defaultOptions, options); } diff --git a/qiniu/cdn.js b/qiniu/cdn.js index 518f5735..75a4ce9b 100644 --- a/qiniu/cdn.js +++ b/qiniu/cdn.js @@ -7,8 +7,8 @@ const encodeUrl = require('encodeurl'); exports.CdnManager = CdnManager; -function CdnManager(mac) { - this.mac = mac || new digest.Mac(); +function CdnManager (mac) { + this.mac = mac || new digest.Mac(); } // 获取域名日志下载链接 @@ -17,21 +17,13 @@ function CdnManager(mac) { // @param domains 域名列表 domains = ['obbid7qc6.qnssl.com','7xkh68.com1.z0.glb.clouddn.com'] // @param logDay 日期，例如 2016-07-01 // @param callbackFunc(err, respBody, respInfo) -CdnManager.prototype.getCdnLogList = function(domains, logDay, callbackFunc) { - var url = '/v2/tune/log/list\n'; - var accessToken = util.generateAccessToken(this.mac, url, ''); - var headers = { - 'Content-Type': 'application/json', - 'Authorization': accessToken, - }; - postBody = { - 'day': logDay, - 'domains': domains.join(';') - } - - req('/v2/tune/log/list', headers, postBody, callbackFunc); -} - +CdnManager.prototype.getCdnLogList = function (domains, logDay, callbackFunc) { + var postBody = { + day: logDay, + domains: domains.join(';') + }; + req(this.mac, '/v2/tune/log/list', postBody, callbackFunc); +}; // 获取域名访问流量数据 // @link http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html#batch-flux @@ -41,25 +33,17 @@ CdnManager.prototype.getCdnLogList = function(domains, logDay, callbackFunc) { // @param granularity 粒度，取值：5min／hour／day // @param domains 域名列表 domain = ['obbid7qc6.qnssl.com','obbid7qc6.qnssl.com']; // @param callbackFunc(err, respBody, respInfo) -CdnManager.prototype.getFluxData = function(startDate, endDate, granularity, - domains, - callbackFunc) { - var url = '/v2/tune/flux\n'; - var accessToken = util.generateAccessToken(this.mac, url, ''); - var headers = { - 'Content-Type': 'application/json', - 'Authorization': accessToken, - }; - data = { - 'startDate': startDate, - 'endDate': endDate, - 'granularity': granularity, - 'domains': domains.join(';') - } - - req('/v2/tune/flux', headers, data, callbackFunc); -} - +CdnManager.prototype.getFluxData = function (startDate, endDate, granularity, + domains, + callbackFunc) { + var data = { + startDate: startDate, + endDate: endDate, + granularity: granularity, + domains: domains.join(';') + }; + req(this.mac, '/v2/tune/flux', data, callbackFunc); +}; // 获取域名访问带宽数据 // @link http://developer.qiniu.com/article/fusion/api/traffic-bandwidth.html @@ -68,86 +52,67 @@ CdnManager.prototype.getFluxData = function(startDate, endDate, granularity, // @param granularity 粒度，取值：5min／hour／day // @param domains 域名列表 domain = ['obbid7qc6.qnssl.com','obbid7qc6.qnssl.com'] // @param callbackFunc(err, respBody, respInfo) -CdnManager.prototype.getBandwidthData = function(startDate, endDate, - granularity, domains, - callbackFunc) { - var url = '/v2/tune/bandwidth\n'; - var accessToken = util.generateAccessToken(this.mac, url, ''); - var headers = { - 'Content-Type': 'application/json', - 'Authorization': accessToken, - }; - data = { - 'startDate': startDate, - 'endDate': endDate, - 'granularity': granularity, - 'domains': domains.join(';') - } - - req('/v2/tune/bandwidth', headers, data, callbackFunc); -} - +CdnManager.prototype.getBandwidthData = function (startDate, endDate, + granularity, domains, + callbackFunc) { + var data = { + startDate: startDate, + endDate: endDate, + granularity: granularity, + domains: domains.join(';') + }; + req(this.mac, '/v2/tune/bandwidth', data, callbackFunc); +}; // 预取文件链接 // @link http://developer.qiniu.com/article/fusion/api/prefetch.html // // @param 预取urls urls = ['http://obbid7qc6.qnssl.com/023','http://obbid7qc6.qnssl.com/025'] // @param callbackFunc(err, respBody, respInfo) -CdnManager.prototype.prefetchUrls = function(urls, callbackFunc) { - var postBody = { - urls: urls - }; - var url = '/v2/tune/prefetch\n'; - var accessToken = util.generateAccessToken(this.mac, url, ''); - var headers = { - 'Content-Type': 'application/json', - 'Authorization': accessToken, - }; - - req('/v2/tune/prefetch', headers, postBody, callbackFunc); -} - +CdnManager.prototype.prefetchUrls = function (urls, callbackFunc) { + var postBody = { + urls: urls + }; + req(this.mac, '/v2/tune/prefetch', postBody, callbackFunc); +}; // 刷新链接 // @link http://developer.qiniu.com/article/fusion/api/refresh.html // 刷新urls refreshUrls = ['http://obbid7qc6.qnssl.com/023','http://obbid7qc6.qnssl.com/025'] -CdnManager.prototype.refreshUrls = function(urls, callbackFunc) { - this.refreshUrlsAndDirs(urls, null, callbackFunc); -} - +CdnManager.prototype.refreshUrls = function (urls, callbackFunc) { + this.refreshUrlsAndDirs(urls, null, callbackFunc); +}; // 刷新目录 // 刷新目录列表，每次最多不可以超过10个目录, 刷新目录需要额外开通权限，可以联系七牛技术支持处理 // @link http://developer.qiniu.com/article/fusion/api/refresh.html // 刷新dirs refreshDirs = ['http://obbid7qc6.qnssl.com/wo/','http://obbid7qc6.qnssl.com/'] -CdnManager.prototype.refreshDirs = function(dirs, callbackFunc) { - this.refreshUrlsAndDirs(null, dirs, callbackFunc); -} - - -CdnManager.prototype.refreshUrlsAndDirs = function(urls, dirs, callbackFunc) { - var postBody = { - urls: urls, - dirs: dirs, - }; - var url = '/v2/tune/refresh\n'; - var accessToken = util.generateAccessToken(this.mac, url, ''); - var headers = { - 'Content-Type': 'application/json', - 'Authorization': accessToken, - }; - - req('/v2/tune/refresh', headers, postBody, callbackFunc); -} - +CdnManager.prototype.refreshDirs = function (dirs, callbackFunc) { + this.refreshUrlsAndDirs(null, dirs, callbackFunc); +}; + +CdnManager.prototype.refreshUrlsAndDirs = function (urls, dirs, callbackFunc) { + var postBody = { + urls: urls, + dirs: dirs + }; + req(this.mac, '/v2/tune/refresh', postBody, callbackFunc); +}; // post 请求 -function req(reqPath, header, reqBody, callbackFunc) { - urllib.request("http://fusion.qiniuapi.com" + reqPath, { - method: 'POST', - headers: header, - data: reqBody, - }, callbackFunc); +function req (mac, reqPath, reqBody, callbackFunc) { + var url = 'http://fusion.qiniuapi.com' + reqPath; + var accessToken = util.generateAccessToken(mac, url, ''); + var headers = { + 'Content-Type': 'application/json', + Authorization: accessToken + }; + urllib.request(url, { + method: 'POST', + headers: headers, + data: reqBody, + dataType: 'json' + }, callbackFunc); } // 构建标准的基于时间戳的防盗链 @@ -158,26 +123,27 @@ function req(reqPath, header, reqBody, callbackFunc) { // @param encryptKey 时间戳防盗链的签名密钥，从七牛后台获取 // @param deadline 链接的有效期时间戳，是以秒为单位的Unix时间戳 // @return signedUrl 最终的带时间戳防盗链的url -CdnManager.prototype.createTimestampAntiLeechUrl = function(domain, fileName, - query, encryptKey, deadline) { - if (query != null) { - urlToSign = domain + '/' + encodeUrl(fileName) + '?' + query; - } else { - urlToSign = domain + '/' + encodeUrl(fileName); - } - - var urlObj = url.parse(urlToSign); - pathname = urlObj.pathname; - - var expireHex = deadline.toString(16); - var signedStr = encryptKey + pathname + expireHex; - - var md5 = crypto.createHash('md5'); - var toSignStr = md5.update(signedStr).digest('hex'); - - if (query != null) { - return urlToSign + '&sign=' + toSignStr + '&t=' + expireHex; - } else { - return urlToSign + '?sign=' + toSignStr + '&t=' + expireHex; - } -} +CdnManager.prototype.createTimestampAntiLeechUrl = function (domain, fileName, + query, encryptKey, deadline) { + var urlToSign; + if (query != null) { + urlToSign = domain + '/' + encodeUrl(fileName) + '?' + query; + } else { + urlToSign = domain + '/' + encodeUrl(fileName); + } + + var urlObj = new url.URL(urlToSign); + var pathname = urlObj.pathname; + + var expireHex = deadline.toString(16); + var signedStr = encryptKey + pathname + expireHex; + + var md5 = crypto.createHash('md5'); + var toSignStr = md5.update(signedStr).digest('hex'); + + if (query != null) { + return urlToSign + '&sign=' + toSignStr + '&t=' + expireHex; + } else { + return urlToSign + '?sign=' + toSignStr + '&t=' + expireHex; + } +}; diff --git a/qiniu/conf.js b/qiniu/conf.js index 44d10372..749037ee 100644 --- a/qiniu/conf.js +++ b/qiniu/conf.js @@ -1,83 +1,117 @@ -const fs = require('fs'); -const path = require('path'); const os = require('os'); const pkg = require('../package.json'); exports.ACCESS_KEY = ''; exports.SECRET_KEY = ''; -var defaultUserAgent = function() { - return 'QiniuNodejs/' + pkg.version + ' (' + os.type() + '; ' + os.platform() + - '; ' + os.arch() + '; )'; -} +var defaultUserAgent = function () { + return 'QiniuNodejs/' + pkg.version + ' (' + os.type() + '; ' + os.platform() + + '; ' + os.arch() + '; )'; +}; exports.USER_AGENT = defaultUserAgent(); -exports.BLOCK_SIZE = 4 * 1024 * 1024; //4MB, never change +exports.BLOCK_SIZE = 4 * 1024 * 1024; // 4MB, never change -//define api form mime type -exports.FormMimeUrl = "application/x-www-form-urlencoded"; -exports.FormMimeJson = "application/json"; -exports.FormMimeRaw = "application/octet-stream"; -exports.RS_HOST = "http://rs.qiniu.com"; -exports.RPC_TIMEOUT = 120000; //120s +// define api form mime type +exports.FormMimeUrl = 'application/x-www-form-urlencoded'; +exports.FormMimeJson = 'application/json'; +exports.FormMimeRaw = 'application/octet-stream'; +exports.RS_HOST = 'rs.qiniu.com'; +exports.RPC_TIMEOUT = 600000; // 600s +let QUERY_REGION_BACKUP_HOSTS = [ + 'uc.qbox.me', + 'api.qiniu.com' +]; +Object.defineProperty(exports, 'QUERY_REGION_BACKUP_HOSTS', { + get: () => QUERY_REGION_BACKUP_HOSTS, + set: v => { + QUERY_REGION_BACKUP_HOSTS = v; + } +}); +let QUERY_REGION_HOST = 'kodo-config.qiniuapi.com'; +Object.defineProperty(exports, 'QUERY_REGION_HOST', { + get: () => QUERY_REGION_HOST, + set: v => { + QUERY_REGION_HOST = v; + QUERY_REGION_BACKUP_HOSTS = []; + } +}); +let UC_HOST = 'uc.qbox.me'; +Object.defineProperty(exports, 'UC_HOST', { + get: () => UC_HOST, + set: v => { + UC_HOST = v; + QUERY_REGION_HOST = v; + QUERY_REGION_BACKUP_HOSTS = []; + } +}); -//proxy +// proxy exports.RPC_HTTP_AGENT = null; exports.RPC_HTTPS_AGENT = null; -exports.Config = function Config(options) { - options = options || {}; - //use http or https protocol - this.useHttpsDomain = options.useHttpsDomain || false; - //use cdn accerlated domains - this.useCdnDomain = options.useCdnDomain || true; - //zone of the bucket - //z0 huadong, z1 huabei, z2 huanan, na0 beimei - this.zone = options.zone || null; - this.zoneExpire = options.zoneExpire || -1; -} +exports.Config = function Config (options) { + options = options || {}; + // use http or https protocol + this.useHttpsDomain = !!(options.useHttpsDomain || false); + // use cdn accerlated domains, this is not work with auto query region + this.useCdnDomain = !!(options.useCdnDomain && true); + // zone of the bucket + // z0 huadong, z1 huabei, z2 huanan, na0 beimei + this.zone = options.zone || null; + this.zoneExpire = options.zoneExpire || -1; + // only available with upload for now + this.regionsProvider = options.regionsProvider || null; +}; -exports.Zone = function(srcUpHosts, cdnUpHosts, ioHost, rsHost, rsfHost, - apiHost) { - this.srcUpHosts = srcUpHosts || {}; - this.cdnUpHosts = cdnUpHosts || {}; - this.ioHost = ioHost || ""; - this.rsHost = rsHost || "rs.qiniu.com"; - this.rsfHost = rsfHost || "rsf.qiniu.com"; - this.apiHost = apiHost || "api.qiniu.com"; - var dotIndex = this.ioHost.indexOf("."); - if (dotIndex != -1) { - var ioTag = this.ioHost.substring(0, dotIndex); - var zoneSepIndex = ioTag.indexOf("-"); - if (zoneSepIndex != -1) { - var zoneTag = ioTag.substring(zoneSepIndex + 1); - switch (zoneTag) { - case "z1": - this.rsHost = "rs-z1.qiniu.com"; - this.rsfHost = "rsf-z1.qiniu.com"; - this.apiHost = "api-z1.qiniu.com"; - break; - case "z2": - this.rsHost = "rs-z2.qiniu.com"; - this.rsfHost = "rsf-z2.qiniu.com"; - this.apiHost = "api-z2.qiniu.com"; - break; - case "na0": - this.rsHost = "rs-na0.qiniu.com"; - this.rsfHost = "rsf-na0.qiniu.com"; - this.apiHost = "api-na0.qiniu.com"; - break; - case "as0": - this.rsHost = "rs-as0.qiniu.com"; - this.rsfHost = "rsf-as0.qiniu.com"; - this.apiHost = "api-as0.qiniu.com"; - break; - default: - this.rsHost = "rs.qiniu.com"; - this.rsfHost = "rsf.qiniu.com"; - this.apiHost = "api.qiniu.com"; - break; - } +exports.Zone = function ( + srcUpHosts, + cdnUpHosts, + ioHost, + rsHost, + rsfHost, + apiHost +) { + this.srcUpHosts = srcUpHosts || []; + this.cdnUpHosts = cdnUpHosts || []; + this.ioHost = ioHost || ''; + this.rsHost = rsHost; + this.rsfHost = rsfHost; + this.apiHost = apiHost; + + // set specific hosts if possible + const dotIndex = this.ioHost.indexOf('.'); + if (dotIndex !== -1) { + const ioTag = this.ioHost.substring(0, dotIndex); + const zoneSepIndex = ioTag.indexOf('-'); + if (zoneSepIndex !== -1) { + const zoneTag = ioTag.substring(zoneSepIndex + 1); + switch (zoneTag) { + case 'z1': + !this.rsHost && (this.rsHost = 'rs-z1.qbox.me'); + !this.rsfHost && (this.rsfHost = 'rsf-z1.qbox.me'); + !this.apiHost && (this.apiHost = 'api-z1.qiniuapi.com'); + break; + case 'z2': + !this.rsHost && (this.rsHost = 'rs-z2.qbox.me'); + !this.rsfHost && (this.rsfHost = 'rsf-z2.qbox.me'); + !this.apiHost && (this.apiHost = 'api-z2.qiniuapi.com'); + break; + case 'na0': + !this.rsHost && (this.rsHost = 'rs-na0.qbox.me'); + !this.rsfHost && (this.rsfHost = 'rsf-na0.qbox.me'); + !this.apiHost && (this.apiHost = 'api-na0.qiniuapi.com'); + break; + case 'as0': + !this.rsHost && (this.rsHost = 'rs-as0.qbox.me'); + !this.rsfHost && (this.rsfHost = 'rsf-as0.qbox.me'); + !this.apiHost && (this.apiHost = 'api-as0.qiniuapi.com'); + break; + } + } } - } -} + + !this.rsHost && (this.rsHost = 'rs.qiniu.com'); + !this.rsfHost && (this.rsfHost = 'rsf.qiniu.com'); + !this.apiHost && (this.apiHost = 'api.qiniuapi.com'); +}; diff --git a/qiniu/fop.js b/qiniu/fop.js index f601a8a4..0ce52026 100644 --- a/qiniu/fop.js +++ b/qiniu/fop.js @@ -2,14 +2,13 @@ const util = require('./util'); const rpc = require('./rpc'); const conf = require('./conf'); const digest = require('./auth/digest'); -const zone = require('./zone.js'); const querystring = require('querystring'); exports.OperationManager = OperationManager; -function OperationManager(mac, config) { - this.mac = mac || new digest.Mac(); - this.config = config || new conf.Config(); +function OperationManager (mac, config) { + this.mac = mac || new digest.Mac(); + this.config = config || new conf.Config(); } // 发送持久化数据处理请求 @@ -21,80 +20,58 @@ function OperationManager(mac, config) { // notifyURL 回调业务服务器，通知处理结果 // force 结果是否强制覆盖已有的同名文件 // @param callbackFunc(err, respBody, respInfo) - 回调函数 -OperationManager.prototype.pfop = function(bucket, key, fops, pipeline, - options, callbackFunc) { - options = options || {}; - var that = this; - //必须参数 - var reqParams = { - bucket: bucket, - key: key, - pipeline: pipeline, - fops: fops.join(";"), - }; +OperationManager.prototype.pfop = function (bucket, key, fops, pipeline, + options, callbackFunc) { + options = options || {}; + // 必须参数 + var reqParams = { + bucket: bucket, + key: key, + pipeline: pipeline, + fops: fops.join(';') + }; - //notifyURL - if (options.notifyURL) { - reqParams.notifyURL = options.notifyURL; - } - - //force - if (options.force) { - reqParams.force = 1; - } - - var useCache = false; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } + // notifyURL + if (options.notifyURL) { + reqParams.notifyURL = options.notifyURL; } - } - if (useCache) { - pfopReq(this.mac, this.config, reqParams, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } + // force + if (options.force) { + reqParams.force = 1; + } - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //pfopReq - pfopReq(that.mac, that.config, reqParams, callbackFunc); + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + pfopReq(ctx.mac, ctx.config, reqParams, callbackFunc); }); - } -} +}; -function pfopReq(mac, config, reqParams, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var requestURI = scheme + config.zone.apiHost + '/pfop/'; - var reqBody = querystring.stringify(reqParams); - var auth = util.generateAccessToken(mac, requestURI, reqBody); - rpc.postWithForm(requestURI, reqBody, auth, callbackFunc); +function pfopReq (mac, config, reqParams, callbackFunc) { + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + config.zone.apiHost + '/pfop/'; + var reqBody = querystring.stringify(reqParams); + var auth = util.generateAccessToken(mac, requestURI, reqBody); + rpc.postWithForm(requestURI, reqBody, auth, callbackFunc); } // 查询持久化数据处理进度 // @param persistentId // @callbackFunc(err, respBody, respInfo) - 回调函数 -OperationManager.prototype.prefop = function(persistentId, callbackFunc) { - var apiHost = "api.qiniu.com"; - if(this.config.zone) { - apiHost=this.config.zone.apiHost; - } - - var scheme = this.config.useHttpsDomain ? "https://" : "http://"; - var requestURI = scheme + apiHost + "/status/get/prefop"; - var reqParams = { - id: persistentId - }; - var reqBody = querystring.stringify(reqParams); - rpc.postWithForm(requestURI, reqBody, null, callbackFunc); -} +OperationManager.prototype.prefop = function (persistentId, callbackFunc) { + var apiHost = 'api.qiniu.com'; + if (this.config.zone) { + apiHost = this.config.zone.apiHost; + } + + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + apiHost + '/status/get/prefop'; + var reqParams = { + id: persistentId + }; + var reqBody = querystring.stringify(reqParams); + rpc.postWithForm(requestURI, reqBody, null, callbackFunc); +}; diff --git a/qiniu/httpc/client.js b/qiniu/httpc/client.js new file mode 100644 index 00000000..d33139c5 --- /dev/null +++ b/qiniu/httpc/client.js @@ -0,0 +1,234 @@ +const http = require('http'); +const https = require('https'); + +const urllib = require('urllib'); + +const middleware = require('./middleware'); +const { ResponseWrapper } = require('./responseWrapper'); + +/** + * + * @param {Object} options + * @param {http.Agent} [options.httpAgent] + * @param {https.Agent} [options.httpsAgent] + * @param {middleware.Middleware[]} [options.middlewares] + * + * @constructor + */ +function HttpClient (options) { + this.httpAgent = options.httpAgent || http.globalAgent; + this.httpsAgent = options.httpsAgent || https.globalAgent; + this.middlewares = options.middlewares || []; +} + +HttpClient.prototype._handleRequest = function (req) { + return new Promise((resolve, reject) => { + try { + urllib.request(req.url, req.urllibOptions, (err, data, resp) => { + if (err) { + err.resp = resp; + reject(err); + return; + } + resolve(new ResponseWrapper({ data, resp })); + }); + } catch (e) { + reject(e); + } + }); +}; + +/** + * Options for request + * @typedef {Object} ReqOpts + * @property {http.Agent} [agent] + * @property {https.Agent} [httpsAgent] + * @property {string} url + * @property {middleware.Middleware[]} middlewares + * @property {urllib.Callback} callback + * @property {urllib.RequestOptions} urllibOptions + */ + +/** + * + * @param {ReqOpts} requestOptions + * @return {Promise} + */ +HttpClient.prototype.sendRequest = function (requestOptions) { + const mwList = this.middlewares.concat(requestOptions.middlewares); + + if (!requestOptions.agent) { + requestOptions.agent = this.httpAgent; + } + + if (!requestOptions.httpsAgent) { + requestOptions.httpsAgent = this.httpsAgent; + } + + const handle = middleware.composeMiddlewares( + mwList, + this._handleRequest + ); + + const resPromise = handle(requestOptions); + + if (requestOptions.callback) { + resPromise + .then(({ data, resp }) => + requestOptions.callback(null, data, resp) + ) + .catch(err => { + requestOptions.callback(err, null, err.resp); + }); + } + + return resPromise; +}; + +/** + * @param {Object} reqOptions + * @param {string} reqOptions.url + * @param {http.Agent} [reqOptions.agent] + * @param {https.Agent} [reqOptions.httpsAgent] + * @param {Object} [reqOptions.params] + * @param {Object} [reqOptions.headers] + * @param {middleware.Middleware[]} [reqOptions.middlewares] + * @param {urllib.RequestOptions} [urllibOptions] + * @return {Promise} + */ +HttpClient.prototype.get = function (reqOptions, urllibOptions) { + const { + url, + params, + headers, + middlewares, + agent, + httpsAgent, + callback + } = reqOptions; + + urllibOptions = urllibOptions || {}; + urllibOptions.method = 'GET'; + urllibOptions.headers = Object.assign( + { + Connection: 'keep-alive' + }, + headers, + urllibOptions.headers || {} + ); + urllibOptions.data = params; + urllibOptions.followRedirect = true; + + return this.sendRequest({ + url: url, + middlewares: middlewares || [], + agent: agent, + httpsAgent: httpsAgent, + callback: callback, + urllibOptions: urllibOptions + }); +}; + +/** + * @param {Object} reqOptions + * @param {string} reqOptions.url + * @param {http.Agent} [reqOptions.agent] + * @param {https.Agent} [reqOptions.httpsAgent] + * @param {string | Buffer | Readable} [reqOptions.data] + * @param {Object} [reqOptions.headers] + * @param {middleware.Middleware[]} [reqOptions.middlewares] + * @param {urllib.RequestOptions} [urllibOptions] + * @return {Promise} + */ +HttpClient.prototype.post = function (reqOptions, urllibOptions) { + const { + url, + data, + headers, + middlewares, + agent, + httpsAgent, + callback + } = reqOptions; + + urllibOptions = urllibOptions || {}; + urllibOptions.method = 'POST'; + urllibOptions.headers = Object.assign( + { + Connection: 'keep-alive' + }, + headers, + urllibOptions.headers || {} + ); + urllibOptions.gzip = true; + + if (Buffer.isBuffer(data) || typeof data === 'string') { + urllibOptions.content = data; + } else if (data) { + urllibOptions.stream = data; + } else { + urllibOptions.headers['Content-Length'] = '0'; + } + + return this.sendRequest({ + url: url, + middlewares: middlewares || [], + agent: agent, + httpsAgent: httpsAgent, + callback: callback, + urllibOptions: urllibOptions + }); +}; + +/** + * @param {Object} reqOptions + * @param {string} reqOptions.url + * @param {http.Agent} [reqOptions.agent] + * @param {https.Agent} [reqOptions.httpsAgent] + * @param {string | Buffer | ReadableStream} [reqOptions.data] + * @param {Object} [reqOptions.headers] + * @param {middleware.Middleware[]} [reqOptions.middlewares] + * @param {urllib.RequestOptions} [urllibOptions] + * @return {Promise} + */ +HttpClient.prototype.put = function (reqOptions, urllibOptions) { + const { + url, + data, + headers, + middlewares, + agent, + httpsAgent, + callback + } = reqOptions; + + urllibOptions = urllibOptions || {}; + urllibOptions.method = 'PUT'; + urllibOptions.headers = Object.assign( + { + Connection: 'keep-alive' + }, + headers, + urllibOptions.headers || {} + ); + urllibOptions.gzip = true; + + if (Buffer.isBuffer(data) || typeof data === 'string') { + urllibOptions.content = data; + } else if (data) { + urllibOptions.stream = data; + } else { + urllibOptions.headers['Content-Length'] = '0'; + } + + return this.sendRequest({ + url: url, + middlewares: middlewares || [], + agent: agent, + httpsAgent: httpsAgent, + callback: callback, + urllibOptions: urllibOptions + }); +}; + +exports.HttpClient = HttpClient; diff --git a/qiniu/httpc/endpoint.js b/qiniu/httpc/endpoint.js new file mode 100644 index 00000000..71308070 --- /dev/null +++ b/qiniu/httpc/endpoint.js @@ -0,0 +1,28 @@ +/** + * @class + * @param {string} host + * @param {Object} [options] + * @param {string} [options.defaultScheme] + * @constructor + */ +function Endpoint (host, options) { + options = options || {}; + + this.host = host; + this.defaultScheme = options.defaultScheme || 'https'; +} + +/** + * @param {Object} [options] + * @param {string} [options.scheme] + */ +Endpoint.prototype.getValue = function (options) { + options = options || {}; + + const scheme = options.scheme || this.defaultScheme; + const host = this.host; + + return scheme + '://' + host; +}; + +exports.Endpoint = Endpoint; diff --git a/qiniu/httpc/endpointsProvider.js b/qiniu/httpc/endpointsProvider.js new file mode 100644 index 00000000..d76df1e2 --- /dev/null +++ b/qiniu/httpc/endpointsProvider.js @@ -0,0 +1,51 @@ +/** + * @interface EndpointsProvider + */ + +/** + * @function + * @name EndpointsProvider#getEndpoints + * @returns {Promise} + */ + +/** + * @interface MutableEndpointsProvider + * @extends EndpointsProvider + */ + +/** + * @function + * @name MutableEndpointsProvider#setEndpoints + * @param {endpoints: Endpoint[]} endpoints + * @returns {Promise} + */ + +// --- could split to files if migrate to typescript --- // + +/** + * @class + * @implements EndpointsProvider + * @property {Endpoint[]} endpoints + * @constructor + * @param {Endpoint[]} endpoints + */ +function StaticEndpointsProvider (endpoints) { + this.endpoints = endpoints; +} + +/** + * @param {Region} region + * @param {string} serviceName + */ +StaticEndpointsProvider.fromRegion = function (region, serviceName) { + return new StaticEndpointsProvider(region.services[serviceName]); +}; + +/** + * @returns {Promise} + */ +StaticEndpointsProvider.prototype.getEndpoints = function () { + return Promise.resolve(this.endpoints); +}; + +exports.StaticEndpointsProvider = StaticEndpointsProvider; diff --git a/qiniu/httpc/middleware/base.js b/qiniu/httpc/middleware/base.js new file mode 100644 index 00000000..163da948 --- /dev/null +++ b/qiniu/httpc/middleware/base.js @@ -0,0 +1,31 @@ +/** + * Middleware could be an interface if migrate to typescript + * @class + * @constructor + */ +function Middleware () { +} + +/** + * @memberOf Middleware + * @param _request + * @param _next + */ +Middleware.prototype.send = function (_request, _next) { + throw new Error('The Middleware NOT be Implemented'); +}; + +exports.Middleware = Middleware; + +/** + * @param {Middleware[]} middlewares + * @param {function(ReqOpts):Promise} handler + * @return {function(ReqOpts):Promise} + */ +exports.composeMiddlewares = function (middlewares, handler) { + return middlewares.reverse() + .reduce( + (h, mw) => request => mw.send(request, h), + handler + ); +}; diff --git a/qiniu/httpc/middleware/index.js b/qiniu/httpc/middleware/index.js new file mode 100644 index 00000000..97c550f0 --- /dev/null +++ b/qiniu/httpc/middleware/index.js @@ -0,0 +1,8 @@ +const base = require('./base'); + +module.exports = { + composeMiddlewares: base.composeMiddlewares, + Middleware: base.Middleware, + RetryDomainsMiddleware: require('./retryDomains').RetryDomainsMiddleware, + UserAgentMiddleware: require('./ua').UserAgentMiddleware +}; diff --git a/qiniu/httpc/middleware/retryDomains.js b/qiniu/httpc/middleware/retryDomains.js new file mode 100644 index 00000000..3bb7d216 --- /dev/null +++ b/qiniu/httpc/middleware/retryDomains.js @@ -0,0 +1,101 @@ +const middleware = require('./base'); + +const URL = require('url').URL; + +/** + * @class + * @extends middleware.Middleware + * @param {Object} retryDomainsOptions + * @param {string[]} retryDomainsOptions.backupDomains + * @param {number} [retryDomainsOptions.maxRetryTimes] + * @param {function(Error || null, ResponseWrapper || null, ReqOpts):boolean} [retryDomainsOptions.retryCondition] + * @constructor + */ +function RetryDomainsMiddleware (retryDomainsOptions) { + this.backupDomains = retryDomainsOptions.backupDomains; + this.maxRetryTimes = retryDomainsOptions.maxRetryTimes || 2; + this.retryCondition = retryDomainsOptions.retryCondition; + + this._retriedTimes = 0; +} + +RetryDomainsMiddleware.prototype = Object.create(middleware.Middleware.prototype); +RetryDomainsMiddleware.prototype.constructor = RetryDomainsMiddleware; + +/** + * @memberOf RetryDomainsMiddleware + * @param {Error || null} err + * @param {ResponseWrapper || null} respWrapper + * @param {ReqOpts} reqOpts + * @return {boolean} + * @private + */ +RetryDomainsMiddleware.prototype._shouldRetry = function (err, respWrapper, reqOpts) { + if (typeof this.retryCondition === 'function') { + return this.retryCondition(err, respWrapper, reqOpts); + } + + return !respWrapper || respWrapper.needRetry(); +}; + +/** + * @memberOf RetryDomainsMiddleware + * @param {ReqOpts} reqOpts + * @param {function(ReqOpts):Promise} next + * @return {Promise} + */ +RetryDomainsMiddleware.prototype.send = function (reqOpts, next) { + const url = new URL(reqOpts.url); + const domains = this.backupDomains.slice(); // copy for late pop + + const couldRetry = () => { + // the reason `this.maxRetryTimes - 1` is request send first then add retriedTimes + // and `this.maxRetryTimes` means max request times per domain + if (this._retriedTimes < this.maxRetryTimes - 1) { + this._retriedTimes += 1; + return true; + } + + if (domains.length) { + this._retriedTimes = 0; + const domain = domains.shift(); + const [hostname, port] = domain.split(':'); + url.hostname = hostname; + url.port = port || url.port; + reqOpts.url = url.toString(); + return true; + } + + return false; + }; + + const tryNext = () => { + return next(reqOpts) + .then(respWrapper => { + if (!this._shouldRetry(null, respWrapper, reqOpts)) { + return respWrapper; + } + + if (couldRetry()) { + return tryNext(); + } + + return respWrapper; + }) + .catch(err => { + if (!this._shouldRetry(err, null, reqOpts)) { + return Promise.reject(err); + } + + if (couldRetry()) { + return tryNext(); + } + + return Promise.reject(err); + }); + }; + + return tryNext(); +}; + +exports.RetryDomainsMiddleware = RetryDomainsMiddleware; diff --git a/qiniu/httpc/middleware/ua.js b/qiniu/httpc/middleware/ua.js new file mode 100644 index 00000000..974ea554 --- /dev/null +++ b/qiniu/httpc/middleware/ua.js @@ -0,0 +1,36 @@ +const os = require('os'); + +const middleware = require('./base'); + +/** + * @class + * @extends middleware.Middleware + * @param {string} sdkVersion + * @constructor + */ +function UserAgentMiddleware (sdkVersion) { + this.userAgent = 'QiniuNodejs/' + sdkVersion + + ' (' + + os.type() + '; ' + + os.platform() + '; ' + + os.arch() + '; ' + + 'Node.js ' + process.version + '; )'; +} +UserAgentMiddleware.prototype = Object.create(middleware.Middleware.prototype); +UserAgentMiddleware.prototype.constructor = UserAgentMiddleware; + +/** + * @memberOf UserAgentMiddleware + * @param {ReqOpts} reqOpts + * @param {function(ReqOpts):Promise} next + * @return {Promise} + */ +UserAgentMiddleware.prototype.send = function (reqOpts, next) { + if (!reqOpts.urllibOptions.headers) { + reqOpts.urllibOptions.headers = {}; + } + reqOpts.urllibOptions.headers['User-Agent'] = this.userAgent; + return next(reqOpts); +}; + +exports.UserAgentMiddleware = UserAgentMiddleware; diff --git a/qiniu/httpc/region.js b/qiniu/httpc/region.js new file mode 100644 index 00000000..297f76df --- /dev/null +++ b/qiniu/httpc/region.js @@ -0,0 +1,184 @@ +const { Endpoint } = require('./endpoint'); + +/** + * @readonly + * @enum {string} + */ +const SERVICE_NAME = { + UC: 'uc', + UP: 'up', + IO: 'io', + RS: 'rs', + RSF: 'rsf', + API: 'api', + S3: 's3' +}; + +// --- could split to files if migrate to typescript --- // + +/** + * @typedef {SERVICE_NAME | string} ServiceKey + */ + +/** + * @param {Object} options + * @param {string} [options.regionId] + * @param {string} [options.s3RegionId] + * @param {Object.} [options.services] + * @param {number} [options.ttl] seconds. default 1 day. + * @param {Date} [options.createTime] + * @constructor + */ +function Region (options) { + this.regionId = options.regionId; + this.s3RegionId = options.s3RegionId || options.regionId; + + this.services = options.services || {}; + // use Object.values when min version of Node.js update to ≥ v7.5.0 + Object.keys(SERVICE_NAME).map(k => { + const v = SERVICE_NAME[k]; + if (!Array.isArray(this.services[v])) { + this.services[v] = []; + } + }); + + this.ttl = options.ttl || 86400; + this.createTime = options.createTime || new Date(); +} + +/** + * This is used to be compatible with Zone. + * So this function will be removed after remove Zone. + * NOTE: The Region instance obtained using this method + * can only be used for the following services: up, io, rs, rsf, api. + * Because the Zone not support other services. + * @param {conf.Zone} zone + * @param {Object} [options] + * @param {string} [options.regionId] + * @param {string} [options.s3RegionId] + * @param {number} [options.ttl] + * @param {boolean} [options.isPreferCdnHost] + */ +Region.fromZone = function (zone, options) { + options = options || {}; + options.ttl = options.ttl || -1; + + const upHosts = options.isPreferCdnHost + ? zone.cdnUpHosts.concat(zone.srcUpHosts) + : zone.srcUpHosts.concat(zone.cdnUpHosts); + + const services = { + // use array destructure if migrate to typescript + [SERVICE_NAME.UP]: upHosts.map( + h => new Endpoint(h) + ), + [SERVICE_NAME.IO]: [ + new Endpoint(zone.ioHost) + ], + [SERVICE_NAME.RS]: [ + new Endpoint(zone.rsHost) + ], + [SERVICE_NAME.RSF]: [ + new Endpoint(zone.rsfHost) + ], + [SERVICE_NAME.API]: [ + new Endpoint(zone.apiHost) + ] + }; + + return new Region({ + regionId: options.regionId, + s3RegionId: options.s3RegionId || options.regionId, + services: services, + ttl: options.ttl + }); +}; + +/** + * @param {string} regionId + * @param {Object} [options] + * @param {string} [options.s3RegionId] + * @param {number} [options.ttl] + * @param {Date} [options.createTime] + * @param {Object.} [options.extendedServices] + * @returns {Region} + */ +Region.fromRegionId = function (regionId, options) { + options = options || {}; + + const s3RegionId = options.s3RegionId || regionId; + const ttl = options.ttl; + const createTime = options.createTime; + + const isZ0 = regionId === 'z0'; + + /** + * @type {Object.} + */ + let services = { + [SERVICE_NAME.UC]: [ + new Endpoint('uc.qiniuapi.com') + ], + [SERVICE_NAME.UP]: isZ0 + ? [ + new Endpoint('upload.qiniup.com'), + new Endpoint('up.qiniup.com'), + new Endpoint('up.qbox.me') + ] + : [ + new Endpoint('upload-' + regionId + '.qiniup.com'), + new Endpoint('up-' + regionId + '.qiniup.com'), + new Endpoint('up-' + regionId + '.qbox.me') + ], + [SERVICE_NAME.IO]: isZ0 + ? [ + new Endpoint('iovip.qiniuio.com'), + new Endpoint('iovip.qbox.me') + ] + : [ + new Endpoint('iovip-' + regionId + '.qiniuio.com'), + new Endpoint('iovip-' + regionId + '.qbox.me') + ], + [SERVICE_NAME.RS]: [ + new Endpoint('rs-' + regionId + '.qiniuapi.com'), + new Endpoint('rs-' + regionId + '.qbox.me') + ], + [SERVICE_NAME.RSF]: [ + new Endpoint('rsf-' + regionId + '.qiniuapi.com'), + new Endpoint('rsf-' + regionId + '.qbox.me') + ], + [SERVICE_NAME.API]: [ + new Endpoint('api-' + regionId + '.qiniuapi.com'), + new Endpoint('api-' + regionId + '.qbox.me') + ], + [SERVICE_NAME.S3]: [ + new Endpoint('s3.' + s3RegionId + '.qiniucs.com') + ] + }; + + services = Object.assign(services, options.extendedServices || {}); + + return new Region({ + regionId: regionId, + s3RegionId: s3RegionId, + services: services, + ttl: ttl, + createTime: createTime + }); +}; + +Object.defineProperty(Region.prototype, 'isLive', { + get: function () { + if (this.ttl < 0) { + return true; + } + // convert ms to s + const liveTime = Math.round((Date.now() - this.createTime) / 1000); + return liveTime < this.ttl; + }, + enumerable: false, + configurable: true +}); + +exports.SERVICE_NAME = SERVICE_NAME; +exports.Region = Region; diff --git a/qiniu/httpc/regionsProvider.js b/qiniu/httpc/regionsProvider.js new file mode 100644 index 00000000..6010d0c1 --- /dev/null +++ b/qiniu/httpc/regionsProvider.js @@ -0,0 +1,697 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const readline = require('readline'); +const stream = require('stream'); + +const { Endpoint } = require('./endpoint'); +const { Region, SERVICE_NAME } = require('./region'); + +/** + * @interface RegionsProvider + */ + +/** + * @function + * @name RegionsProvider#getRegions + * @returns {Promise} + */ + +/** + * @interface MutableRegionsProvider + * @extends RegionsProvider + */ + +/** + * @function + * @name MutableRegionsProvider#setRegions + * @param {Region[]} regions + * @returns {Promise} + */ + +// --- could split to files if migrate to typescript --- // + +/** + * @class + * @implements RegionsProvider + * @param {Region[]} regions + * @constructor + */ +function StaticRegionsProvider (regions) { + this.regions = regions; +} + +StaticRegionsProvider.prototype.getRegions = function () { + return Promise.resolve(this.regions); +}; + +// --- could split to files if migrate to typescript --- // +const CachedRegionsProvider = (function () { + /** + * cache regions in memory. + * @private DO NOT export this. + * @type {Map} + */ + const memoCachedRegions = new Map(); + const lastShrinkAt = new Date(0); + + /** + * @class + * @implements MutableRegionsProvider + * @param {Object} [options] + * @param {string} options.cacheKey + * @param {RegionsProvider} options.baseRegionsProvider + * @param {number} [options.shrinkInterval] + * @param {string} [options.persistPath] + * @constructor + */ + function CachedRegionsProvider ( + options + ) { + // only used for testing + this._memoCache = memoCachedRegions; + + this.cacheKey = options.cacheKey; + this.baseRegionsProvider = options.baseRegionsProvider; + + this.lastShrinkAt = lastShrinkAt; + this.shrinkInterval = options.shrinkInterval || 86400 * 1000; + this.persistPath = options.persistPath; + // allow disable persist + if (!this.persistPath && this.persistPath !== null) { + this.persistPath = path.join(os.tmpdir(), 'qn-regions-cache.jsonl'); + } + } + + /** + * @returns {Promise} + */ + CachedRegionsProvider.prototype.getRegions = function () { + /** @type Region[] */ + return shrinkCache.call(this) + .then(() => { + const getRegionsFns = [ + getRegionsFromMemo, + getRegionsFromFile, + getRegionsFromBaseProvider + ]; + + return getRegionsFns.reduce((promiseChain, getRegionsFn) => { + return promiseChain.then(regions => { + if (regions.length) { + return regions; + } + return getRegionsFn.call(this); + }); + }, Promise.resolve([])); + }); + }; + + /** + * @param {Region[]} regions + * @returns {Promise} + */ + CachedRegionsProvider.prototype.setRegions = function (regions) { + this._memoCache.set(this.cacheKey, regions); + if (!this.persistPath) { + return Promise.resolve(); + } + return new Promise(resolve => { + fs.appendFile( + this.persistPath, + stringifyPersistedRegions( + this.cacheKey, + regions + ) + os.EOL, + err => { + if (err) { + resolve(); + return; + } + resolve(); + } + ); + }); + }; + + /** + * the returns value means if shrunk or not. + * @private + * @returns {Promise} + */ + function shrinkCache () { + const now = new Date(); + const shouldShrink = this.lastShrinkAt.getTime() + this.shrinkInterval < now.getTime(); + if (!shouldShrink) { + return Promise.resolve(false); + } + this.lastShrinkAt = now; + + // shrink memory cache + for (const [key, regions] of this._memoCache.entries()) { + const liveRegions = regions.filter(r => r.isLive); + if (liveRegions.length) { + this._memoCache.set(key, liveRegions); + } else { + this._memoCache.delete(key); + } + } + + if (!this.persistPath) { + return Promise.resolve(true); + } + // shrink file cache + const shrunkCache = new Map(); + const shrinkPath = this.persistPath + '.shrink'; + const lockPath = this.persistPath + '.shrink.lock'; + const unlockShrink = () => { + try { + fs.unlinkSync(lockPath); + } catch (err) { + if (err.code !== 'ENOENT') { + console.error(err); + } + } + }; + return new Promise((resolve, reject) => { + // lock to shrink + fs.open(lockPath, 'wx', (err, fd) => { + if (err) { + reject(err); + return; + } + fs.closeSync(fd); + resolve(); + }); + // prevent deadlock if exit unexpectedly when shrinking + process.on('exit', unlockShrink); + }) + .then(() => { + // parse useless data + return walkFileCache.call(this, ({ + cacheKey, + regions + }) => { + const validRegions = regions.filter(r => r.isLive); + if (!validRegions.length) { + return; + } + + if (!shrunkCache.has(cacheKey)) { + shrunkCache.set(cacheKey, validRegions); + return; + } + + const shrunkRegions = shrunkCache.get(cacheKey); + shrunkCache.set( + cacheKey, + mergeRegions(shrunkRegions, validRegions) + ); + }); + }) + .then(() => { + // write to file + const shrunkCacheIterator = shrunkCache.entries(); + const cacheReadStream = new stream.Readable({ + read: function () { + const nextEntry = shrunkCacheIterator.next(); + if (nextEntry.done) { + this.push(null); + } else { + const [cacheKey, regions] = nextEntry.value; + this.push( + stringifyPersistedRegions( + cacheKey, + regions + ) + os.EOL + ); + } + } + }); + const writeStream = fs.createWriteStream(shrinkPath); + return new Promise((resolve, reject) => { + const pipeline = cacheReadStream.pipe(writeStream); + pipeline.on('close', resolve); + pipeline.on('error', reject); + }); + }) + .then(() => { + return new Promise(resolve => { + fs.rename(shrinkPath, this.persistPath, () => resolve()); + }); + }) + .then(() => { + unlockShrink(); + process.removeListener('exit', unlockShrink); + return Promise.resolve(true); + }) + .catch(err => { + // if exist + if (err.code === 'EEXIST' && err.path === lockPath) { + // ignore file shrinking err + return Promise.resolve(false); + } + // use finally when min version of Node.js update to ≥ v10.3.0 + unlockShrink(); + process.removeListener('exit', unlockShrink); + return Promise.reject(err); + }); + } + + /** + * @private + * @returns {Promise} + */ + function getRegionsFromMemo () { + const regions = this._memoCache.get(this.cacheKey); + + if (Array.isArray(regions) && regions.length) { + return Promise.resolve(regions); + } + + return Promise.resolve([]); + } + + /** + * @returns {Promise} + */ + function getRegionsFromFile () { + if (!this.persistPath) { + return Promise.resolve([]); + } + + return flushFileCacheToMemo.call(this) + .then(() => { + return getRegionsFromMemo.call(this); + }) + .catch(() => { + return Promise.resolve([]); + }); + } + + /** + * @returns {Promise} + */ + function getRegionsFromBaseProvider () { + return this.baseRegionsProvider.getRegions() + .then(regions => { + if (regions.length) { + return this.setRegions(regions); + } + return Promise.resolve(); + }) + .then(() => { + return getRegionsFromMemo.call(this); + }); + } + + /** + * @private + * @param {function(CachedPersistedRegions):void} fn + * @param {Object} [options] + * @param {boolean} [options.ignoreParseError] + * @return {Promise} + */ + function walkFileCache (fn, options) { + options = options || {}; + options.ignoreParseError = options.ignoreParseError || false; + if (!fs.existsSync(this.persistPath)) { + return Promise.resolve(); + } + return new Promise((resolve, reject) => { + const rl = readline.createInterface({ + input: fs.createReadStream(this.persistPath) + }); + + rl + .on('line', (line) => { + try { + const cachedPersistedRegions = parsePersistedRegions(line); + fn(cachedPersistedRegions); + } catch (err) { + if (!options.ignoreParseError) { + rl.close(); + reject(err); + } + } + }) + .on('close', () => { + resolve(); + }); + }); + } + + /** + * @private + * @returns Promise + */ + function flushFileCacheToMemo () { + return walkFileCache.call(this, ({ + cacheKey, + regions + }) => { + const validRegions = regions.filter(r => r.isLive); + if (!validRegions.length) { + return; + } + + if (!this._memoCache.has(cacheKey)) { + this._memoCache.set(cacheKey, validRegions); + return; + } + + const memoRegions = this._memoCache.get(cacheKey); + this._memoCache.set( + cacheKey, + mergeRegions(memoRegions, validRegions) + ); + }); + } + + // --- serializers --- + + /** + * @typedef EndpointPersistInfo + * @property {string} host + * @property {string} defaultScheme + */ + + /** + * @param {Endpoint} endpoint + * @returns {EndpointPersistInfo} + */ + function persistEndpoint (endpoint) { + return { + defaultScheme: endpoint.defaultScheme, + host: endpoint.host + }; + } + + /** + * @param {EndpointPersistInfo} persistInfo + * @returns {Endpoint} + */ + function getEndpointFromPersisted (persistInfo) { + return new Endpoint(persistInfo.host, { + defaultScheme: persistInfo.defaultScheme + }); + } + + /** + * @typedef RegionPersistInfo + * @property {string} [regionId] + * @property {string} s3RegionId + * @property {Object.} services + * @property {number} ttl + * @property {number} createTime + */ + + /** + * @param {Region} region + * @returns {RegionPersistInfo} + */ + function persistRegion (region) { + /** + * @type {Object.} + */ + const persistedServices = {}; + // use Object.entries when min version of Node.js update to ≥ v7.5.0 + for (const k of Object.keys(region.services)) { + const v = region.services[k]; + persistedServices[k] = v.map(persistEndpoint); + } + + return { + regionId: region.regionId, + s3RegionId: region.s3RegionId, + services: persistedServices, + ttl: region.ttl, + createTime: region.createTime.getTime() + }; + } + + /** + * @param {RegionPersistInfo} persistInfo + * @returns {Region} + */ + function getRegionFromPersisted (persistInfo) { + /** + * @param {EndpointPersistInfo[]} servicePersistEndpoint + * @returns {Endpoint[]} + */ + const convertToEndpoints = (servicePersistEndpoint) => { + // The `persistInfo` is from disk that may be broken. + if (!Array.isArray(servicePersistEndpoint)) { + return []; + } + + return servicePersistEndpoint.map(getEndpointFromPersisted); + }; + + /** + * @type {Object.} + */ + const services = {}; + for (const serviceName of Object.keys(persistInfo.services)) { + const endpointPersistInfos = persistInfo.services[serviceName]; + services[serviceName] = convertToEndpoints(endpointPersistInfos); + } + + return new Region({ + regionId: persistInfo.regionId, + s3RegionId: persistInfo.s3RegionId, + services: services, + ttl: persistInfo.ttl, + createTime: new Date(persistInfo.createTime) + }); + } + + /** + * @typedef CachedPersistedRegions + * @property {string} cacheKey + * @property {Region[]} regions + */ + + /** + * @private + * @param {string} persistedRegions + * @return {CachedPersistedRegions} + */ + function parsePersistedRegions (persistedRegions) { + const { cacheKey, regions } = JSON.parse(persistedRegions); + return { + cacheKey, + regions: regions.map(getRegionFromPersisted) + }; + } + + /** + * @private + * @param {string} cacheKey + * @param {Region[]} regions + * @return {string} + */ + function stringifyPersistedRegions (cacheKey, regions) { + return JSON.stringify({ + cacheKey, + regions: regions.map(persistRegion) + }); + } + + /** + * merge two regions by region id. + * if the same region id, the last create region will be keep. + * @param {Region[]} regionsA + * @param {Region[]} regionsB + * @returns {Region[]} + */ + function mergeRegions (regionsA, regionsB) { + if (!regionsA.length) { + return regionsB; + } + if (!regionsB.length) { + return regionsA; + } + + const convertRegionsToMap = (regions) => regions.reduce((m, r) => { + if ( + m[r.regionId] && + m[r.regionId].createTime > r.createTime + ) { + return m; + } + m[r.regionId] = r; + return m; + }, {}); + + const regionsMapA = convertRegionsToMap(regionsA); + const regionsMapB = convertRegionsToMap(regionsB); + + // union region ids + const regionIds = new Set(); + Object.keys(regionsMapA).forEach(rid => regionIds.add(rid)); + Object.keys(regionsMapB).forEach(rid => regionIds.add(rid)); + + // merge + const result = []; + for (const regionId of regionIds) { + if (regionsMapA[regionId] && regionsMapB[regionId]) { + if (regionsMapA[regionId].createTime > regionsMapB[regionId].createTime) { + result.push(regionsMapA[regionId]); + } else { + result.push(regionsMapB[regionId]); + } + } else { + if (regionsMapA[regionId]) { + result.push(regionsMapA[regionId]); + } else if (regionsMapB[regionId]) { + result.push(regionsMapB[regionId]); + } + } + } + return result; + } + + return CachedRegionsProvider; +})(); +// --- could split to files if migrate to typescript --- // +const { RetryDomainsMiddleware } = require('../httpc/middleware'); +const rpc = require('../rpc'); + +const QueryRegionsProvider = (function () { + /** + * @class + * @implements RegionsProvider + * @param {Object} options + * @param {string} options.accessKey + * @param {string} options.bucketName + * @param {EndpointsProvider} options.endpointsProvider + * @constructor + */ + function QueryRegionsProvider (options) { + this.accessKey = options.accessKey; + this.bucketName = options.bucketName; + this.endpintsProvider = options.endpointsProvider; + } + + /** + * @return {Promise} + */ + QueryRegionsProvider.prototype.getRegions = function () { + return this.endpintsProvider.getEndpoints() + .then(endpoints => { + const [preferredEndpoint, ...alternativeEndpoints] = endpoints; + + if (!preferredEndpoint) { + return Promise.reject(new Error('There isn\'t available endpoints to query regions')); + } + + const middlewares = []; + if (alternativeEndpoints.length) { + middlewares.push( + new RetryDomainsMiddleware({ + backupDomains: alternativeEndpoints.map(e => e.host) + }) + ); + } + + const url = preferredEndpoint.getValue() + '/v4/query'; + + // send request; + return rpc.qnHttpClient.get({ + url: url, + params: { + ak: this.accessKey, + bucket: this.bucketName + }, + middlewares: middlewares + }); + }) + .then(respWrapper => { + if (!respWrapper.ok()) { + return Promise.reject( + new Error('Query regions failed with HTTP Status Code' + respWrapper.resp.statusCode) + ); + } + try { + const hosts = JSON.parse(respWrapper.data).hosts; + return hosts.map(getRegionFromQuery); + } catch (err) { + return Promise.reject( + new Error('There isn\'t available hosts in query result', { + cause: err + }) + ); + } + }); + }; + + /** + * @param {Object} data + * @param {string} data.region + * @param {Object} data.s3 + * @param {string[]} data.s3.domains + * @param {string} data.s3.region_alias + * @param {Object} data.uc + * @param {string[]} data.uc.domains + * @param {Object} data.up + * @param {string[]} data.up.domains + * @param {Object} data.io + * @param {string[]} data.io.domains + * @param {Object} data.rs + * @param {string[]} data.rs.domains + * @param {Object} data.rsf + * @param {string[]} data.rsf.domains + * @param {Object} data.api + * @param {string[]} data.api.domains + * @param {number} data.ttl + * @returns {Region} + */ + function getRegionFromQuery (data) { + /** + * @param {string[]} domains + * @returns {Endpoint[]} + */ + const convertToEndpoints = (domains) => { + if (!Array.isArray(domains)) { + return []; + } + return domains.map(d => new Endpoint(d)); + }; + + let services = { + [SERVICE_NAME.UC]: convertToEndpoints(data.uc.domains), + [SERVICE_NAME.UP]: convertToEndpoints(data.up.domains), + [SERVICE_NAME.IO]: convertToEndpoints(data.io.domains), + [SERVICE_NAME.RS]: convertToEndpoints(data.rs.domains), + [SERVICE_NAME.RSF]: convertToEndpoints(data.rsf.domains), + [SERVICE_NAME.API]: convertToEndpoints(data.api.domains), + [SERVICE_NAME.S3]: convertToEndpoints(data.s3.domains) + }; + + // forward compatibility with new services + services = Object.keys(data) + // use Object.entries when min version of Node.js update to ≥ v7.5.0 + .map(k => ([k, data[k]])) + .reduce((s, [k, v]) => { + if (v && Array.isArray(v.domains) && !(k in s)) { + s[k] = convertToEndpoints(v.domains); + } + return s; + }, services); + + return new Region({ + regionId: data.region, + s3RegionId: data.s3.region_alias, + services: services, + ttl: data.ttl, + createTime: new Date() + }); + } + + return QueryRegionsProvider; +})(); + +exports.StaticRegionsProvider = StaticRegionsProvider; +exports.CachedRegionsProvider = CachedRegionsProvider; +exports.QueryRegionsProvider = QueryRegionsProvider; diff --git a/qiniu/httpc/responseWrapper.js b/qiniu/httpc/responseWrapper.js new file mode 100644 index 00000000..75f50951 --- /dev/null +++ b/qiniu/httpc/responseWrapper.js @@ -0,0 +1,34 @@ +function ResponseWrapper ({ + data, + resp +}) { + this.data = data; + this.resp = resp; +} + +/** + * @return {boolean} + */ +ResponseWrapper.prototype.ok = function () { + return this.resp && Math.floor(this.resp.statusCode / 100) === 2; +}; + +/** + * @return {boolean} + */ +ResponseWrapper.prototype.needRetry = function () { + if (this.resp.statusCode > 0 && this.resp.statusCode < 500) { + return false; + } + + // https://developer.qiniu.com/fusion/kb/1352/the-http-request-return-a-status-code + if ([ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ].includes(this.resp.statusCode)) { + return false; + } + + return true; +}; + +exports.ResponseWrapper = ResponseWrapper; diff --git a/qiniu/rpc.js b/qiniu/rpc.js index 41683d6d..4ecff28f 100644 --- a/qiniu/rpc.js +++ b/qiniu/rpc.js @@ -1,80 +1,237 @@ -var urllib = require('urllib'); -var util = require('./util'); -var conf = require('./conf'); +const urllib = require('urllib'); +const pkg = require('../package.json'); +const conf = require('./conf'); +const digest = require('./auth/digest'); +const util = require('./util'); +const client = require('./httpc/client'); +const middleware = require('./httpc/middleware'); + +let uaMiddleware = new middleware.UserAgentMiddleware(pkg.version); +uaMiddleware = Object.defineProperty(uaMiddleware, 'userAgent', { + get: function () { + return conf.USER_AGENT; + } +}); +exports.qnHttpClient = new client.HttpClient({ + middlewares: [ + uaMiddleware + ] +}); +exports.get = get; exports.post = post; +exports.put = put; +exports.getWithOptions = getWithOptions; +exports.getWithToken = getWithToken; +exports.postWithOptions = postWithOptions; exports.postMultipart = postMultipart; exports.postWithForm = postWithForm; exports.postWithoutForm = postWithoutForm; -function postMultipart(requestURI, requestForm, callbackFunc) { - return post(requestURI, requestForm, requestForm.headers(), callbackFunc); +function addAuthHeaders (headers, mac) { + const xQiniuDate = util.formatDateUTC(new Date(), 'YYYYMMDDTHHmmssZ'); + if (mac.options.disableQiniuTimestampSignature !== null) { + if (!mac.options.disableQiniuTimestampSignature) { + headers['X-Qiniu-Date'] = xQiniuDate; + } + } else if (process.env.DISABLE_QINIU_TIMESTAMP_SIGNATURE) { + if (process.env.DISABLE_QINIU_TIMESTAMP_SIGNATURE.toLowerCase() !== 'true') { + headers['X-Qiniu-Date'] = xQiniuDate; + } + } else { + headers['X-Qiniu-Date'] = xQiniuDate; + } + return headers; +} + +function getWithOptions (requestURI, options, callbackFunc) { + let headers = options.headers || {}; + const mac = options.mac || new digest.Mac(); + + if (!headers['Content-Type']) { + headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } + + headers = addAuthHeaders(headers, mac); + + // if there are V3, V4 token generator in the future, extends options with signVersion + const token = util.generateAccessTokenV2( + mac, + requestURI, + 'GET', + headers['Content-Type'], + null, + headers + ); + + if (mac.accessKey) { + headers.Authorization = token; + } + + return get(requestURI, headers, callbackFunc); +} + +function getWithToken (requestUrl, token, callbackFunc) { + const headers = { + 'Content-Type': 'application/x-www-form-urlencoded' + }; + if (token) { + headers.Authorization = token; + } + return get(requestUrl, headers, callbackFunc); +} + +function postWithOptions (requestURI, requestForm, options, callbackFunc) { + let headers = options.headers || {}; + const mac = options.mac || new digest.Mac(); + + if (!headers['Content-Type']) { + headers['Content-Type'] = 'application/x-www-form-urlencoded'; + } + + headers = addAuthHeaders(headers, mac); + + // if there are V3, V4 token generator in the future, extends options with signVersion + const token = util.generateAccessTokenV2( + mac, + requestURI, + 'POST', + headers['Content-Type'], + requestForm, + headers + ); + + if (mac.accessKey) { + headers.Authorization = token; + } + + return post(requestURI, requestForm, headers, callbackFunc); +} + +function postMultipart (requestURI, requestForm, callbackFunc) { + return post(requestURI, requestForm, requestForm.headers(), callbackFunc); } -function postWithForm(requestURI, requestForm, token, callbackFunc) { - var headers = { - 'Content-Type': 'application/x-www-form-urlencoded' - }; - if (token) { - headers['Authorization'] = token; - } - return post(requestURI, requestForm, headers, callbackFunc); +function postWithForm (requestURI, requestForm, token, callbackFunc) { + const headers = { + 'Content-Type': 'application/x-www-form-urlencoded' + }; + if (token) { + headers.Authorization = token; + } + return post(requestURI, requestForm, headers, callbackFunc); } -function postWithoutForm(requestURI, token, callbackFunc) { - var headers = { - 'Content-Type': 'application/x-www-form-urlencoded', - }; - if (token) { - headers['Authorization'] = token; - } - return post(requestURI, null, headers, callbackFunc); +function postWithoutForm (requestURI, token, callbackFunc) { + const headers = { + 'Content-Type': 'application/x-www-form-urlencoded' + }; + if (token) { + headers.Authorization = token; + } + return post(requestURI, null, headers, callbackFunc); } -function post(requestURI, requestForm, headers, callbackFunc) { - //var start = parseInt(Date.now() / 1000); - headers = headers || {}; - headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; - headers['Connection'] = 'keep-alive'; - - var data = { - headers: headers, - method: 'POST', - dataType: 'json', - timeout: conf.RPC_TIMEOUT, - gzip: true, - // timing: true, - }; - - if (conf.RPC_HTTP_AGENT) { - data['agent'] = conf.RPC_HTTP_AGENT; - } - - if (conf.RPC_HTTPS_AGENT) { - data['httpsAgent'] = conf.RPC_HTTPS_AGENT; - } - - if (Buffer.isBuffer(requestForm) || typeof requestForm === 'string') { - data.content = requestForm; - } else if (requestForm) { - data.stream = requestForm; - } else { - data.headers['Content-Length'] = 0; - }; - - var req = urllib.request(requestURI, data, function(respErr, respBody, - respInfo) { - //var end = parseInt(Date.now() / 1000); - // console.log((end - start) + " seconds"); - // console.log("queuing:\t" + respInfo.timing.queuing); - // console.log("dnslookup:\t" + respInfo.timing.dnslookup); - // console.log("connected:\t" + respInfo.timing.connected); - // console.log("requestSent:\t" + respInfo.timing.requestSent); - // console.log("waiting:\t" + respInfo.timing.waiting); - // console.log("contentDownload:\t" + respInfo.timing.contentDownload); - - callbackFunc(respErr, respBody, respInfo); - }); - - return req; +function get (requestUrl, headers, callbackFunc) { + headers = headers || {}; + headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; + headers.Connection = 'keep-alive'; + + const data = { + method: 'GET', + headers: headers, + dataType: 'json', + timeout: conf.RPC_TIMEOUT, + gzip: true + }; + + if (conf.RPC_HTTP_AGENT) { + data.agent = conf.RPC_HTTP_AGENT; + } + + if (conf.RPC_HTTPS_AGENT) { + data.httpsAgent = conf.RPC_HTTPS_AGENT; + } + + return urllib.request( + requestUrl, + data, + callbackFunc + ); +} + +function post (requestUrl, requestForm, headers, callbackFunc) { + // var start = parseInt(Date.now() / 1000); + headers = headers || {}; + headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; + headers.Connection = 'keep-alive'; + + const data = { + headers: headers, + method: 'POST', + dataType: 'json', + timeout: conf.RPC_TIMEOUT, + gzip: true + // timing: true, + }; + + if (conf.RPC_HTTP_AGENT) { + data.agent = conf.RPC_HTTP_AGENT; + } + + if (conf.RPC_HTTPS_AGENT) { + data.httpsAgent = conf.RPC_HTTPS_AGENT; + } + + if (Buffer.isBuffer(requestForm) || typeof requestForm === 'string') { + data.content = requestForm; + } else if (requestForm) { + data.stream = requestForm; + } else { + data.headers['Content-Length'] = 0; + } + + return urllib.request( + requestUrl, + data, + callbackFunc + ); +} + +function put (requestUrl, requestForm, headers, callbackFunc) { + // var start = parseInt(Date.now() / 1000); + headers = headers || {}; + headers['User-Agent'] = headers['User-Agent'] || conf.USER_AGENT; + headers.Connection = 'keep-alive'; + + const data = { + headers: headers, + method: 'PUT', + dataType: 'json', + timeout: conf.RPC_TIMEOUT, + gzip: true + // timing: true, + }; + + if (conf.RPC_HTTP_AGENT) { + data.agent = conf.RPC_HTTP_AGENT; + } + + if (conf.RPC_HTTPS_AGENT) { + data.httpsAgent = conf.RPC_HTTPS_AGENT; + } + + if (Buffer.isBuffer(requestForm) || typeof requestForm === 'string') { + data.content = requestForm; + } else if (requestForm) { + data.stream = requestForm; + } else { + data.headers['Content-Length'] = 0; + } + + return urllib.request( + requestUrl, + data, + callbackFunc + ); } diff --git a/qiniu/rtc/app.js b/qiniu/rtc/app.js new file mode 100644 index 00000000..7da91791 --- /dev/null +++ b/qiniu/rtc/app.js @@ -0,0 +1,123 @@ +var http = require('http'); + +const host = 'rtc.qiniuapi.com'; +const headers = { + 'Content-Type': 'application/json' +}; + +function get(credentials, options, fn) { + options.headers.Authorization = credentials.generateAccessToken(options, null); + + var req = http.request(options, function (res) { + res.setEncoding('utf-8'); + + var responseString = ''; + + res.on('data', function (data) { + responseString += data; + }); + + res.on('end', function () { + var resultObject = JSON.parse(responseString); + + if (res.statusCode != 200) { + var result = { + code: res.statusCode, + message: resultObject.error || res.statusMessage, + reqId: res.headers['x-reqid'] + }; + fn(result, null); + } else { + fn(null, resultObject); + } + }); + }); + + req.on('error', function (e) { + fn(e, null); + }); + + req.end(); +} + +function post(credentials, options, data, fn) { + var dataString = JSON.stringify(data); + + options.headers.Authorization = credentials.generateAccessToken(options, dataString); + + var req = http.request(options, function (res) { + res.setEncoding('utf-8'); + + var responseString = ''; + + res.on('data', function (data) { + responseString += data; + }); + + res.on('end', function () { + var resultObject = JSON.parse(responseString); + + if (res.statusCode != 200) { + var result = { + code: res.statusCode, + message: resultObject.error || res.statusMessage, + reqId: res.headers['x-reqid'] + }; + fn(result, null); + } else { + fn(null, resultObject); + } + }); + }); + req.on('error', function (e) { + fn(e, null); + }); + + req.write(dataString); + + req.end(); +} + +exports.createApp = function (app, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps', + method: 'POST', + headers: headers + }; + post(credentials, options, app, fn); +}; + +exports.getApp = function (appId, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps/' + appId, + method: 'GET', + headers: headers + }; + get(credentials, options, fn); +}; + +exports.deleteApp = function (appId, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps/' + appId, + method: 'DELETE', + headers: headers + }; + get(credentials, options, fn); +}; + +exports.updateApp = function (appId, app, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps/' + appId, + method: 'POST', + headers: headers + }; + post(credentials, options, app, fn); +}; diff --git a/qiniu/rtc/credentials.js b/qiniu/rtc/credentials.js new file mode 100644 index 00000000..9b9c12b8 --- /dev/null +++ b/qiniu/rtc/credentials.js @@ -0,0 +1,57 @@ +var util = require('./util'); + +function Credentials (accessKey, secretKey) { + this.accessKey = accessKey; + this.secretKey = secretKey; +} + +Credentials.prototype.generateAccessToken = function (options, data) { + var sign = this._signRequest(options, data); + var token = 'Qiniu' + ' ' + this.accessKey + ':' + sign; + + return token; +}; + +Credentials.prototype._signRequest = function (options, body) { + var contentType = options.headers['Content-Type']; + + var host = options.host; + if (options.port && options.port != 80) { + host = host + ':' + options.port; + } + + var data = options.method + ' ' + options.path; + data += '\nHost: ' + host; + if (contentType) { + data += '\nContent-Type: ' + contentType; + } + data += '\n\n'; + + if (body && contentType && contentType != 'application/octet-stream') { + data += body; + } + + var digest = util.hmacSha1(data, this.secretKey); + + var sageDigest = util.base64ToUrlSafe(digest); + + return sageDigest; +}; + +Credentials.prototype.sign = function (data) { + var digest = util.hmacSha1(data, this.secretKey); + var sageDigest = util.base64ToUrlSafe(digest); + return this.accessKey + ':' + sageDigest; +}; + +Credentials.prototype.signJson = function (opt) { + var str = JSON.stringify(opt); + var encodedStr = util.urlsafeBase64Encode(str); + var sign = util.hmacSha1(encodedStr, this.secretKey); + var encodedSign = util.base64ToUrlSafe(sign); + + var token = this.accessKey + ':' + encodedSign + ':' + encodedStr; + return token; +}; + +module.exports = exports = Credentials; diff --git a/qiniu/rtc/room.js b/qiniu/rtc/room.js new file mode 100644 index 00000000..dc6a8999 --- /dev/null +++ b/qiniu/rtc/room.js @@ -0,0 +1,118 @@ +var http = require('http'); + +const host = 'rtc.qiniuapi.com'; +const headers = { + 'Content-Type': 'application/json' +}; + +function get (credentials, options, fn) { + options.headers.Authorization = credentials.generateAccessToken(options, null); + + var req = http.request(options, function (res) { + res.setEncoding('utf-8'); + + var responseString = ''; + + res.on('data', function (data) { + responseString += data; + }); + + res.on('end', function () { + // var resultObject = JSON.parse(responseString); + // console.log(JSON.parse(responseString)) + + if (res.statusCode != 200) { + var result = { + code: res.statusCode, + message: res.statusMessage + }; + fn(result, null); + } else { + fn(null, JSON.parse(responseString)); + } + }); + }); + + req.on('error', function (e) { + fn(e, null); + }); + + req.end(); +} + +// function post(credentials, options, data, fn) { +// var dataString = JSON.stringify(data); + +// options.headers['Authorization'] = credentials.generateAccessToken(options, dataString); + +// var req = http.request(options, function(res) { +// res.setEncoding('utf-8'); + +// var responseString = ''; + +// res.on('data', function(data) { +// responseString += data; +// }); + +// res.on('end', function() { +// var resultObject = JSON.parse(responseString); + +// if (res.statusCode != 200) { +// var result = { +// code: res.statusCode, +// message: res.statusMessage +// }; +// fn(result, null); +// } else { +// fn(null, resultObject); +// } +// }); +// }); +// req.on('error', function(e) { +// fn(e, null); +// }); + +// req.write(dataString); + +// req.end(); +// } + +exports.listUser = function (appId, roomName, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps/' + appId + '/rooms/' + roomName + '/users', + method: 'GET', + headers: headers + }; + get(credentials, options, fn); +}; + +exports.kickUser = function (appId, roomName, userId, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps/' + appId + '/rooms/' + roomName + '/users/' + userId, + method: 'DELETE', + headers: headers + }; + get(credentials, options, fn); +}; + +exports.listActiveRooms = function (appId, roomNamePrefix, offset, limit, credentials, fn) { + var options = { + host: host, + port: 80, + path: '/v3/apps/' + appId + '/rooms?prefix=' + roomNamePrefix + '&offset=' + offset + '&limit=' + limit, + method: 'GET', + headers: headers + }; + get(credentials, options, fn); +}; + +exports.getRoomToken = function (roomAccess, credentials) { + if (!roomAccess.expireAt) { + roomAccess.expireAt = Math.floor(Date.now() / 1000) + 3600; + } + return credentials.signJson(roomAccess); +}; diff --git a/qiniu/rtc/util.js b/qiniu/rtc/util.js new file mode 100644 index 00000000..dec4c362 --- /dev/null +++ b/qiniu/rtc/util.js @@ -0,0 +1,16 @@ +var crypto = require('crypto'); + +exports.base64ToUrlSafe = function (v) { + return v.replace(/\//g, '_').replace(/\+/g, '-'); +}; + +exports.urlsafeBase64Encode = function (jsonFlags) { + var encoded = Buffer.from(jsonFlags).toString('base64'); + return exports.base64ToUrlSafe(encoded); +}; + +exports.hmacSha1 = function (encodedFlags, secretKey) { + var hmac = crypto.createHmac('sha1', secretKey); + hmac.update(encodedFlags); + return hmac.digest('base64'); +}; diff --git a/qiniu/sms/message.js b/qiniu/sms/message.js new file mode 100644 index 00000000..fbcfc595 --- /dev/null +++ b/qiniu/sms/message.js @@ -0,0 +1,58 @@ +const util = require('../util'); +const urllib = require('urllib'); +exports.sendMessage = function (reqBody,mac,callbackFunc){ + reqBody = JSON.stringify(reqBody); + var args = { + requestURI:"https://sms.qiniuapi.com/v1/message", + reqBody:reqBody, + mac:mac, + } + post(args,callbackFunc); +} + +exports.sendSingleMessage = function (reqBody,mac,callbackFunc){ + reqBody = JSON.stringify(reqBody); + var args = { + requestURI:"https://sms.qiniuapi.com/v1/message/single", + reqBody:reqBody, + mac:mac, + } + post(args,callbackFunc); +} + +exports.sendOverseaMessage = function (reqBody,mac,callbackFunc){ + reqBody = JSON.stringify(reqBody); + var args = { + requestURI:"https://sms.qiniuapi.com/v1/message/oversea", + reqBody:reqBody, + mac:mac, + } + post(args,callbackFunc); +} + +exports.sendFulltextMessage = function (reqBody,mac,callbackFunc){ + reqBody = JSON.stringify(reqBody); + var args = { + requestURI:"https://sms.qiniuapi.com/v1/message/fulltext", + reqBody:reqBody, + mac:mac, + } + post(args,callbackFunc); +} + +function post(args,callbackFunc){ + var contentType = 'application/json'; + var accessToken = util.generateAccessTokenV2(args.mac, args.requestURI, 'POST', contentType, args.reqBody); + var headers = { + 'Authorization': accessToken, + 'Content-Type': contentType, + } + + var data = { + method: 'POST', + headers: headers, + data: args.reqBody, + } + + urllib.request(args.requestURI, data, callbackFunc); +} diff --git a/qiniu/storage/form.js b/qiniu/storage/form.js index e0b6bf2b..2ef4cdb5 100644 --- a/qiniu/storage/form.js +++ b/qiniu/storage/form.js @@ -1,189 +1,405 @@ -const conf = require('../conf'); -const util = require('../util'); -const rpc = require('../rpc'); const fs = require('fs'); -const getCrc32 = require('crc32'); const path = require('path'); -const mime = require('mime'); const Readable = require('stream').Readable; + +const getCrc32 = require('crc32'); +const mime = require('mime'); const formstream = require('formstream'); -const zone = require('../zone'); -const digest = require('../auth/digest'); + +const conf = require('../conf'); +const util = require('../util'); +const rpc = require('../rpc'); + +const { + prepareRegionsProvider, + doWorkWithRetry, + ChangeEndpointRetryPolicy, + ChangeRegionRetryPolicy +} = require('./internal'); exports.FormUploader = FormUploader; exports.PutExtra = PutExtra; -function FormUploader(config) { - this.config = config || new conf.Config(); +/** + * @class + * @param {conf.Config} [config] + * @constructor + */ +function FormUploader (config) { + this.config = config || new conf.Config(); + + // RetryPolicy API sign isn't stable not export to user + // Internal usage only + this.retryPolicies = [ + new ChangeEndpointRetryPolicy(), + new ChangeRegionRetryPolicy() + ]; } -// 上传可选参数 -// @params fname 请求体中的文件的名称 -// @params params 额外参数设置，参数名称必须以x:开头 -// @param mimeType 指定文件的mimeType -// @param crc32 指定文件的crc32值 -// @param checkCrc 指定是否检测文件的crc32值 -function PutExtra(fname, params, mimeType, crc32, checkCrc) { - this.fname = fname || ''; - this.params = params || {}; - this.mimeType = mimeType || null; - this.crc32 = crc32 || null; - this.checkCrc = checkCrc || 1; +/** + * 上传可选参数 + * @param {string} [fname] 请求体中的文件的名称 + * @param {Object} [params] 额外参数设置，参数名称必须以x:开头 + * @param {string} [mimeType] 指定文件的mimeType + * @param {string} [crc32] 指定文件的crc32值 + * @param {number | boolean} [checkCrc] 指定是否检测文件的crc32值 + * @param {Object} [metadata] 元数据设置，参数名称必须以 x-qn-meta-${name}: 开头 + */ +function PutExtra ( + fname, + params, + mimeType, + crc32, + checkCrc, + metadata +) { + this.fname = fname || ''; + this.params = params || {}; + this.mimeType = mimeType || null; + this.crc32 = crc32 || null; + this.checkCrc = checkCrc || true; + this.metadata = metadata || {}; } -FormUploader.prototype.putStream = function(uploadToken, key, fsStream, - putExtra, callbackFunc) { - putExtra = putExtra || new PutExtra(); - if (!putExtra.mimeType) { - putExtra.mimeType = 'application/octet-stream'; - } - - if (!putExtra.fname) { - putExtra.fname = key ? key : 'fname'; - } - - fsStream.on("error", function(err) { - //callbackFunc - callbackFunc(err, null, null); - return; - }); - - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - var accessKey = util.getAKFromUptoken(uploadToken); - var bucket = util.getBucketFromUptoken(uploadToken); - if (useCache) { - createMultipartForm(uploadToken, key, fsStream, putExtra, function( - postForm) { - putReq(that.config, postForm, callbackFunc); - }); - } else { - zone.getZoneInfo(accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } +/** + * @callback reqCallback + * + * @param { Error } err + * @param { Object } ret + * @param { http.IncomingMessage } info + */ - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; +/** + * @typedef UploadResult + * @property {any} data + * @property {http.IncomingMessage} resp + */ - //req - createMultipartForm(uploadToken, key, fsStream, +/** + * @param {string} uploadToken + * @param {string | null} key + * @param {stream.Readable} fsStream + * @param {PutExtra | null} putExtra + * @param {reqCallback} callbackFunc + * @returns {Promise} + */ +FormUploader.prototype.putStream = function ( + uploadToken, + key, + fsStream, + putExtra, + callbackFunc +) { + const preferScheme = this.config.useHttpsDomain ? 'https' : 'http'; + + // PutExtra + putExtra = getDefaultPutExtra( putExtra, - function(postForm) { - putReq(that.config, postForm, callbackFunc); - }); + { + key + } + ); + + fsStream.on('error', function (err) { + callbackFunc(err, null, null); }); - } + + // RegionsProvider + return prepareRegionsProvider({ + config: this.config, + bucketName: util.getBucketFromUptoken(uploadToken), + accessKey: util.getAKFromUptoken(uploadToken) + }) + .then(regionsProvider => { + return doWorkWithRetry({ + workFn: sendPutReq, + + callbackFunc, + regionsProvider, + // stream not support retry + retryPolicies: [] + }); + }); + + function sendPutReq (endpoint) { + const endpointValue = endpoint.getValue({ + scheme: preferScheme + }); + + const postForm = createMultipartForm( + uploadToken, + key, + fsStream, + putExtra + ); + return new Promise(resolve => { + putReq( + endpointValue, + postForm, + (err, ret, info) => resolve({ err, ret, info }) + ); + }); + } +}; + +/** + * @param {string} upDomain + * @param {formstream} postForm + * @param {reqCallback} callbackFunc + */ +function putReq (upDomain, postForm, callbackFunc) { + rpc.postMultipart(upDomain, postForm, callbackFunc); } -function putReq(config, postForm, callbackFunc) { - //set up hosts order - var upHosts = []; +/** + * 上传字节 + * @param {string} uploadToken + * @param {string | null} key + * @param {any} body + * @param {PutExtra | null} putExtra + * @param {reqCallback} callbackFunc + * @returns {Promise} + */ +FormUploader.prototype.put = function ( + uploadToken, + key, + body, + putExtra, + callbackFunc +) { + const preferScheme = this.config.useHttpsDomain ? 'https' : 'http'; + + // initial PutExtra + putExtra = getDefaultPutExtra( + putExtra, + { + key + } + ); + + // initial RegionsProvider + return prepareRegionsProvider({ + config: this.config, + bucketName: util.getBucketFromUptoken(uploadToken), + accessKey: util.getAKFromUptoken(uploadToken) + }) + .then(regionsProvider => { + return doWorkWithRetry({ + workFn: sendPutReq, - if (config.useCdnDomain) { - if (config.zone.cdnUpHosts) { - config.zone.cdnUpHosts.forEach(function(host) { - upHosts.push(host); - }); + callbackFunc, + regionsProvider, + retryPolicies: this.retryPolicies + }); + }); + + function sendPutReq (endpoint) { + const fsStream = new Readable(); + fsStream.push(body); + fsStream.push(null); + + const endpointValue = endpoint.getValue({ + scheme: preferScheme + }); + + const postForm = createMultipartForm( + uploadToken, + key, + fsStream, + putExtra + ); + + return new Promise(resolve => { + putReq( + endpointValue, + postForm, + (err, ret, info) => { + resolve({ err, ret, info }); + } + ); + }); } - config.zone.srcUpHosts.forEach(function(host) { - upHosts.push(host); - }); - } else { - config.zone.srcUpHosts.forEach(function(host) { - upHosts.push(host); +}; + +/** + * @param {string} uploadToken + * @param {any} body + * @param {PutExtra | null} putExtra + * @param {reqCallback} callbackFunc + * @returns {Promise} + */ +FormUploader.prototype.putWithoutKey = function ( + uploadToken, + body, + putExtra, + callbackFunc +) { + return this.put(uploadToken, null, body, putExtra, callbackFunc); +}; + +/** + * @param {string} uploadToken + * @param {string | null} key + * @param {stream.Readable} fsStream + * @param {PutExtra | null} putExtra + * @returns {formstream} + */ +function createMultipartForm (uploadToken, key, fsStream, putExtra) { + const postForm = formstream(); + postForm.field('token', uploadToken); + if (key != null) { + postForm.field('key', key); + } + // fix the bug of formstream + // https://html.spec.whatwg.org/#multipart-form-data + const escapeFname = putExtra.fname.replace(/"/g, '%22') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A'); + postForm.stream( + 'file', + fsStream, + escapeFname, + putExtra.mimeType + ); + + // putExtra params + for (const k in putExtra.params) { + if (k.startsWith('x:')) { + postForm.field(k, putExtra.params[k].toString()); + } + } + + // putExtra metadata + for (const metadataKey in putExtra.metadata) { + if (metadataKey.startsWith('x-qn-meta-')) { + postForm.field(metadataKey, putExtra.metadata[metadataKey].toString()); + } + } + + let fileBody = []; + fsStream.on('data', function (data) { + fileBody.push(data); }); - config.zone.cdnUpHosts.forEach(function(host) { - upHosts.push(host); + + fsStream.on('end', function () { + if (putExtra.checkCrc) { + if (putExtra.crc32 == null) { + fileBody = Buffer.concat(fileBody); + const bodyCrc32 = parseInt('0x' + getCrc32(fileBody)); + postForm.field('crc32', bodyCrc32.toString()); + } else { + postForm.field('crc32', putExtra.crc32); + } + } }); - } - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var upDomain = scheme + upHosts[0]; - rpc.postMultipart(upDomain, postForm, callbackFunc); + return postForm; } -// 上传字节 -// -FormUploader.prototype.put = function(uploadToken, key, body, putExtra, - callbackFunc) { - var fsStream = new Readable(); - fsStream.push(body); - fsStream.push(null); - - putExtra = putExtra || new PutExtra(); - return this.putStream(uploadToken, key, fsStream, putExtra, callbackFunc) -} +/** 上传本地文件 + * @param {string} uploadToken 上传凭证 + * @param {string | null} key 目标文件名 + * @param {string} localFile 本地文件路径 + * @param {PutExtra | null} putExtra 额外选项 + * @param callbackFunc 回调函数 + * @returns {Promise} + */ +FormUploader.prototype.putFile = function ( + uploadToken, + key, + localFile, + putExtra, + callbackFunc +) { + const preferScheme = this.config.useHttpsDomain ? 'https' : 'http'; -FormUploader.prototype.putWithoutKey = function(uploadToken, body, putExtra, - callbackFunc) { - return this.put(uploadToken, null, body, putExtra, callbackFunc); -} + // initial PutExtra + putExtra = putExtra || new PutExtra(); + if (!putExtra.mimeType) { + putExtra.mimeType = mime.getType(localFile); + } -function createMultipartForm(uploadToken, key, fsStream, putExtra, callbackFunc) { - var postForm = formstream(); - postForm.field('token', uploadToken); - if (key) { - postForm.field('key', key); - } - postForm.stream('file', fsStream, putExtra.fname, putExtra.mimeType); - - //putExtra params - for (var k in putExtra.params) { - if (k.startsWith("x:")) { - postForm.field(k, putExtra.params[k].toString()); + if (!putExtra.fname) { + putExtra.fname = path.basename(localFile); } - } - var fileBody = []; - fsStream.on('data', function(data) { - fileBody.push(data); - }); - - fsStream.on('end', function() { - fileBody = Buffer.concat(fileBody); - var bodyCrc32 = parseInt("0x" + getCrc32(fileBody)); - postForm.field('crc32', bodyCrc32); - }); - callbackFunc(postForm); -} + putExtra = getDefaultPutExtra( + putExtra, + { + key + } + ); + + // initial RegionsProvider + return prepareRegionsProvider({ + config: this.config, + bucketName: util.getBucketFromUptoken(uploadToken), + accessKey: util.getAKFromUptoken(uploadToken) + }) + .then(regionsProvider => { + return doWorkWithRetry({ + workFn: sendPutReq, -// 上传本地文件 -// @params uploadToken 上传凭证 -// @param key 目标文件名 -// @param localFile 本地文件路径 -// @param putExtra 额外选项 -// @param callbackFunc 回调函数 -FormUploader.prototype.putFile = function(uploadToken, key, localFile, putExtra, - callbackFunc) { - putExtra = putExtra || new PutExtra(); - var fsStream = fs.createReadStream(localFile); + callbackFunc, + regionsProvider, + retryPolicies: this.retryPolicies + }); + }); + + function sendPutReq (endpoint) { + const fsStream = fs.createReadStream(localFile); + const endpointValue = endpoint.getValue({ + scheme: preferScheme + }); + const postForm = createMultipartForm( + uploadToken, + key, + fsStream, + putExtra + ); + return new Promise(resolve => { + putReq( + endpointValue, + postForm, + (err, ret, info) => { + resolve({ err, ret, info }); + } + ); + }); + } +}; - if (!putExtra.mimeType) { - putExtra.mimeType = mime.lookup(localFile); - } +/** 上传本地文件 + * @param {string} uploadToken 上传凭证 + * @param {string} localFile 本地文件路径 + * @param {PutExtra | null} putExtra 额外选项 + * @param callbackFunc 回调函数 + * @returns {Promise} + */ +FormUploader.prototype.putFileWithoutKey = function ( + uploadToken, + localFile, + putExtra, + callbackFunc +) { + return this.putFile(uploadToken, null, localFile, putExtra, callbackFunc); +}; - if (!putExtra.fname) { - putExtra.fname = path.basename(localFile); - } +/** + * @param {PutExtra} putExtra + * @param {Object} options + * @param {string} options.key + * @return {PutExtra} + */ +function getDefaultPutExtra (putExtra, options) { + putExtra = putExtra || new PutExtra(); + if (!putExtra.mimeType) { + putExtra.mimeType = 'application/octet-stream'; + } - return this.putStream(uploadToken, key, fsStream, putExtra, callbackFunc); -} + if (!putExtra.fname) { + putExtra.fname = options.key || 'fname'; + } -FormUploader.prototype.putFileWithoutKey = function(uploadToken, localFile, - putExtra, callbackFunc) { - return this.putFile(uploadToken, null, localFile, putExtra, callbackFunc); + return putExtra; } diff --git a/qiniu/storage/internal.js b/qiniu/storage/internal.js new file mode 100644 index 00000000..5bdbc937 --- /dev/null +++ b/qiniu/storage/internal.js @@ -0,0 +1,600 @@ +// internal +// DO NOT use this file, unless you know what you're doing. +// Because its API may make broken change for internal usage. + +const conf = require('../conf'); +const fs = require('fs'); + +const { + Region, SERVICE_NAME +} = require('../httpc/region'); +const { + CachedRegionsProvider, + QueryRegionsProvider, + StaticRegionsProvider +} = require('../httpc/regionsProvider'); +const { + Endpoint +} = require('../httpc/endpoint'); +const { + StaticEndpointsProvider +} = require('../httpc/endpointsProvider'); +const { ResponseWrapper } = require('../httpc/responseWrapper'); +const crypto = require('crypto'); + +exports.prepareRegionsProvider = prepareRegionsProvider; +exports.doWorkWithRetry = doWorkWithRetry; +exports.ChangeEndpointRetryPolicy = ChangeEndpointRetryPolicy; +exports.ChangeRegionRetryPolicy = ChangeRegionRetryPolicy; +exports.TokenExpiredRetryPolicy = TokenExpiredRetryPolicy; + +/** + * @param {string} [defaultScheme] + * @returns {StaticEndpointsProvider} + */ +function getDefaultQueryRegionEndpointsProvider (defaultScheme) { + defaultScheme = defaultScheme || 'https'; + + /** + * @type {string[]} + */ + const queryRegionHosts = [conf.QUERY_REGION_HOST].concat(conf.QUERY_REGION_BACKUP_HOSTS); + + return new StaticEndpointsProvider(queryRegionHosts.map(h => new Endpoint(h, { defaultScheme }))); +} + +/** + * @param {Object} options + * @param {string} options.accessKey + * @param {string} options.bucketName + * @param {EndpointsProvider} [options.queryRegionsEndpointProvider] + * @param {string} [options.defaultScheme] + * @returns {Promise} + */ +function getDefaultRegionsProvider (options) { + let queryRegionsEndpointProvider = options.queryRegionsEndpointProvider; + if (!queryRegionsEndpointProvider) { + queryRegionsEndpointProvider = getDefaultQueryRegionEndpointsProvider(options.defaultScheme); + } + + return queryRegionsEndpointProvider.getEndpoints() + .then(endpoints => { + const endpointsMd5 = endpoints + .map(e => e.host) + .sort() + .reduce( + (hash, host) => hash.update(host), + crypto.createHash('md5') + ) + .digest('hex'); + const cacheKey = [ + endpointsMd5, + options.accessKey, + options.bucketName + ].join(':'); + return new CachedRegionsProvider({ + cacheKey, + baseRegionsProvider: new QueryRegionsProvider({ + accessKey: options.accessKey, + bucketName: options.bucketName, + endpointsProvider: new StaticEndpointsProvider(endpoints) + }) + }); + }); +} + +/** + * @param {Object} options + * @param {conf.Config} options.config + * @param {string} options.bucketName + * @param {string} options.accessKey + * @returns {Promise} + */ +function prepareRegionsProvider (options) { + const { + config, + bucketName, + accessKey + } = options; + + // prepare RegionsProvider + let regionsProvider = config.regionsProvider; + if (regionsProvider) { + return Promise.resolve(regionsProvider); + } + + // backward compatibility with zone + let zoneTtl; + let shouldUseZone; + if (config.zoneExpire > 0) { + zoneTtl = config.zoneExpire - Math.trunc(Date.now() / 1000); + shouldUseZone = config.zone && zoneTtl > 0; + } else { + zoneTtl = -1; + shouldUseZone = Boolean(config.zone); + } + if (shouldUseZone) { + regionsProvider = new StaticRegionsProvider([ + Region.fromZone(config.zone, { + ttl: zoneTtl, + isPreferCdnHost: config.useCdnDomain + }) + ]); + } + if (regionsProvider) { + return Promise.resolve(regionsProvider); + } + + return getDefaultRegionsProvider({ + accessKey, + bucketName, + defaultScheme: config.useHttpsDomain ? 'https' : 'http' + }); +} + +// --- split to files --- // + +/** + * @interface RetryPolicy + */ + +/** + * should have an options arguments for receiving work params like ReqOptions to initial context + * @function + * @name RetryPolicy#initContext + * @param {Object} context + * @returns {Promise} + */ + +/** + * @typedef RetryRet + * @property {any} data + * @property {IncomingMessage} resp + */ + +/** + * @function + * @name RetryPolicy#prepareRetry + * @param {Object} context + * @param {RetryRet} ret + * @returns {Promise} + */ + +// --- split to files --- // + +/** + * @class + * @constructor + * @implements RetryPolicy + * @param {Object} [options] + * @param {number} [options.maxRetryTimes] + */ +function TokenExpiredRetryPolicy (options) { + options = options || {}; + this.id = Symbol(this.constructor.name); + this.maxRetryTimes = options.maxRetryTimes || 1; +} + +/** + * @param {string} resumeRecordFilePath + * @returns {boolean} + */ +TokenExpiredRetryPolicy.prototype.isResumedUpload = function (resumeRecordFilePath) { + if (!resumeRecordFilePath) { + return false; + } + return fs.existsSync(resumeRecordFilePath); +}; + +/** + * @param {Object} context + * @returns {Promise} + */ +TokenExpiredRetryPolicy.prototype.initContext = function (context) { + context[this.id] = { + retriedTimes: 0 + }; + return Promise.resolve(); +}; + +/** + * @param {Object} context + * @param {RetryRet} ret + * @return {boolean} + */ +TokenExpiredRetryPolicy.prototype.shouldRetry = function (context, ret) { + const { + resumeRecordFilePath, + uploadApiVersion + } = context; + const { + retriedTimes + } = context[this.id]; + + if ( + retriedTimes >= this.maxRetryTimes || + !this.isResumedUpload(resumeRecordFilePath) + ) { + return false; + } + + if (!ret) { + return false; + } + + if (uploadApiVersion === 'v1' && + ret.resp.statusCode === 701 + ) { + return true; + } + + if (uploadApiVersion === 'v2' && + ret.resp.statusCode === 612 + ) { + return true; + } + + return false; +}; + +/** + * @param {Object} context + * @param {RetryRet} ret + * @returns {Promise} + */ +TokenExpiredRetryPolicy.prototype.prepareRetry = function (context, ret) { + if (!this.shouldRetry(context, ret)) { + return Promise.resolve(false); + } + context[this.id].retriedTimes += 1; + return new Promise(resolve => { + if (!context.resumeRecordFilePath) { + resolve(true); + return; + } + fs.unlink(context.resumeRecordFilePath, _err => { + resolve(true); + }); + }); +}; + +/** + * @class + * @implements RetryPolicy + * @constructor + */ +function ChangeEndpointRetryPolicy () { +} + +/** + * @param {Object} context + * @returns {Promise} + */ +ChangeEndpointRetryPolicy.prototype.initContext = function (context) { + context.alternativeEndpoints = context.alternativeEndpoints || []; + return Promise.resolve(); +}; + +/** + * @param {Object} context + * @param {RetryRet} _ret + * @return {boolean} + */ +ChangeEndpointRetryPolicy.prototype.shouldRetry = function (context, _ret) { + return context.alternativeEndpoints.length > 0; +}; + +/** + * @param {Object} context + * @param {RetryRet} ret + * @return {Promise} + */ +ChangeEndpointRetryPolicy.prototype.prepareRetry = function (context, ret) { + if (!this.shouldRetry(context, ret)) { + return Promise.resolve(false); + } + context.endpoint = context.alternativeEndpoints.shift(); + return Promise.resolve(true); +}; + +/** + * @class + * @constructor + * @implements RetryPolicy + */ +function ChangeRegionRetryPolicy () { +} + +/** + * @param {Object} context + * @returns {Promise} + */ +ChangeRegionRetryPolicy.prototype.initContext = function (context) { + context.alternativeRegions = context.alternativeRegions || []; + return Promise.resolve(); +}; + +/** + * @param {Object} context + * @param {RetryRet} _ret + * @returns {boolean} + */ +ChangeRegionRetryPolicy.prototype.shouldRetry = function (context, _ret) { + return context.alternativeRegions.length > 0; +}; + +/** + * @param {Object} context + * @param {RetryRet} ret + * @returns {Promise} + */ +ChangeRegionRetryPolicy.prototype.prepareRetry = function (context, ret) { + if (!this.shouldRetry(context, ret)) { + return Promise.resolve(false); + } + + const { + resumeRecordFilePath, + serviceName + } = context; + + // resume upload change region + if (resumeRecordFilePath) { + try { + fs.unlinkSync(resumeRecordFilePath); + } catch (_e) { + // ignore + } + } + + // normal change region + context.region = context.alternativeRegions.shift(); + return StaticEndpointsProvider.fromRegion( + context.region, + serviceName + ) + .getEndpoints() + .then(([endpoint, ...alternativeEndpoints]) => { + context.endpoint = endpoint; + context.alternativeEndpoints = alternativeEndpoints; + return Promise.resolve(true); + }); +}; + +/** + * @class + * @constructor + * @param {Object} options + * @param {RetryPolicy[]} [options.retryPolicies] + * @param {string} [options.resumeRecordFilePath] + * @param {RegionsProvider} options.regionsProvider + * @param {'v1' | 'v2' | string} options.uploadApiVersion + * @param {EndpointsProvider} [options.preferredEndpointsProvider] + */ +function UploadState (options) { + this.retryPolicies = options.retryPolicies || []; + this.regionsProvider = options.regionsProvider; + this.preferredEndpointsProvider = options.preferredEndpointsProvider; + this.context = { + serviceName: SERVICE_NAME.UP, + uploadApiVersion: options.uploadApiVersion, + resumeRecordFilePath: options.resumeRecordFilePath + }; +} + +/** + * @returns {Promise} + */ +UploadState.prototype.init = function () { + /** + * loop regions try to find the first region with at least one endpoint + * @returns {Promise} + */ + const loopRegions = () => { + const endpointProvider = StaticEndpointsProvider.fromRegion( + this.context.region, + this.context.serviceName + ); + return endpointProvider.getEndpoints() + .then(endpoints => { + [this.context.endpoint, ...this.context.alternativeEndpoints] = endpoints; + // check endpoint available and change to next region if not + if (this.context.endpoint) { + return; + } + if (!this.context.alternativeRegions.length) { + return Promise.reject(new Error( + 'There isn\'t available endpoint of ' + + this.context.serviceName + + ' service in any regions' + )); + } + this.context.region = this.context.alternativeRegions.shift(); + return loopRegions(); + }); + }; + let preferredEndpoints; + return Promise.resolve() + .then(() => { + if (this.preferredEndpointsProvider) { + return this.preferredEndpointsProvider.getEndpoints(); + } + return []; + }) + .then(endpoints => { + preferredEndpoints = endpoints; + return this.regionsProvider.getRegions(); + }) + .then(regions => { + regions = regions.slice(); + // find preferred region by preferred endpoints + let preferredRegionIndex = -1; + if (preferredEndpoints.length) { + preferredRegionIndex = regions.findIndex(r => + r.services[this.context.serviceName].some(e => + preferredEndpoints.map(pe => pe.host).includes(e.host) + ) + ); + } + // preferred endpoints is not a region, then make all regions alternative + if (preferredEndpoints.length && preferredRegionIndex < 0) { + [this.context.endpoint, ...this.context.alternativeEndpoints] = preferredEndpoints; + this.context.alternativeRegions = regions; + return Promise.resolve(); + } + // preferred endpoints is a region, then reorder the regions + if (preferredEndpoints.length && preferredRegionIndex > 0) { + [this.context.region] = regions.splice(preferredRegionIndex, 1); + this.context.alternativeRegions = regions; + } else { + [this.context.region, ...this.context.alternativeRegions] = regions; + } + // check region available + if (!this.context.region) { + return Promise.reject(new Error('There isn\'t available region')); + } + return loopRegions(); + }) + .then(() => { + // initial all retry policies + return this.retryPolicies.reduce( + (promiseChain, retrier) => { + return promiseChain.then(() => retrier.initContext(this.context)); + }, + Promise.resolve() + ); + }); +}; + +/** + * @param {RetryRet} ret + * @returns {Promise} + */ +UploadState.prototype.prepareRetry = function (ret) { + let [retryPolicy, ...alternativeRetryPolicies] = this.retryPolicies; + const loopRetryPolicies = () => { + if (!retryPolicy) { + return Promise.resolve(false); + } + return retryPolicy.prepareRetry(this.context, ret) + .then(readyToRetry => { + if (readyToRetry) { + return true; + } + retryPolicy = alternativeRetryPolicies.shift(); + if (!retryPolicy) { + return false; + } + return loopRetryPolicies(); + }); + }; + return loopRetryPolicies(); +}; + +/** + * @callback WorkFn + * @param {Endpoint} endpoint + * @returns {Promise<{ err: Error, ret: any, info: IncomingMessage }>} + */ + +/** + * @callback ReqcallbackFunc + * @param {Error} err + * @param {any} ret + * @param {http.IncomingMessage} info + */ + +/** + * @param options + * @param {WorkFn} options.workFn + * @param {ReqcallbackFunc} [options.callbackFunc] + * @param {RegionsProvider} options.regionsProvider + * @param {RetryPolicy[]} [options.retryPolicies] + * @param {'v1' | 'v2' | string} [options.uploadApiVersion] + * @param {string} [options.resumeRecordFilePath] + * @param {EndpointsProvider} [options.preferredEndpointsProvider] + * @returns {Promise} + */ +function doWorkWithRetry (options) { + const workFn = options.workFn; + + const callbackFunc = options.callbackFunc; + const isValidCallback = typeof callbackFunc === 'function'; + const regionsProvider = options.regionsProvider; + const retryPolicies = options.retryPolicies || []; + const uploadApiVersion = options.uploadApiVersion; + const resumeRecordFilePath = options.resumeRecordFilePath; + const preferredEndpointsProvider = options.preferredEndpointsProvider; + + const uploadState = new UploadState({ + retryPolicies, + regionsProvider, + uploadApiVersion, + resumeRecordFilePath, + preferredEndpointsProvider + }); + + // the workFn helper used for recursive calling to retry + const workFnWithRetry = () => { + return workFn(uploadState.context.endpoint) + .then(resp => { + const { + ret, + info + } = resp; + const respWrapper = new ResponseWrapper({ + data: ret, + resp: info + }); + if (!respWrapper.needRetry()) { + return resp; + } + return uploadState.prepareRetry({ data: ret, resp: info }) + .then(readyToRetry => { + if (readyToRetry) { + return workFnWithRetry(); + } + return resp; + }); + }); + }; + + return uploadState.init() + .then(() => { + return workFnWithRetry(); + }) + // change Promise style resolve object for more ease refactoring to RespWrapper in future + .then(({ err, ret, info }) => { + if (err) { + return Promise.reject(err); + } + try { + isValidCallback && callbackFunc(null, ret, info); + } catch (e) { + warningCallbackError(e); + } + return Promise.resolve({ data: ret, resp: info }); + }) + .catch(err => { + // `info` doesn't pass to callback by legacy, could be improved in the future. + try { + isValidCallback && callbackFunc(err, null, null); + } catch (e) { + warningCallbackError(e); + } + return Promise.reject(err); + }); +} + +/** + * @param {Error} e + */ +function warningCallbackError (e) { + console.warn( + 'WARNING:\n' + + 'qiniu SDK will migrate API to Promise style gradually.\n' + + 'The callback style will not be removed for now,\n' + + 'but you should catch your error in your callback function itself' + ); + console.error(e); +} diff --git a/qiniu/storage/resume.js b/qiniu/storage/resume.js index b03829d0..48b692ee 100644 --- a/qiniu/storage/resume.js +++ b/qiniu/storage/resume.js @@ -1,273 +1,843 @@ -const conf = require('../conf'); -const zone = require('../zone'); -const util = require('../util'); -const rpc = require('../rpc'); +const fs = require('fs'); const path = require('path'); + const mime = require('mime'); -const fs = require('fs'); const getCrc32 = require('crc32'); +const destroy = require('destroy'); +const BlockStream = require('block-stream2'); + +const conf = require('../conf'); +const util = require('../util'); +const rpc = require('../rpc'); + +const { + prepareRegionsProvider, + doWorkWithRetry, + TokenExpiredRetryPolicy, + ChangeEndpointRetryPolicy, + ChangeRegionRetryPolicy +} = require('./internal'); +const { StaticEndpointsProvider } = require('../httpc/endpointsProvider'); +const { Endpoint } = require('../httpc/endpoint'); exports.ResumeUploader = ResumeUploader; exports.PutExtra = PutExtra; -function ResumeUploader(config) { - this.config = config || new conf.Config(); +/** + * @param {conf.Config} [config] + * @constructor + */ +function ResumeUploader (config) { + this.config = config || new conf.Config(); + + /** + * Internal usage only for now. + * @readonly + */ + this.retryPolicies = [ + new TokenExpiredRetryPolicy(), + new ChangeEndpointRetryPolicy(), + new ChangeRegionRetryPolicy() + ]; } -// 上传可选参数 -// @params fname 请求体中的文件的名称 -// @params params 额外参数设置，参数名称必须以x:开头 -// @param mimeType 指定文件的mimeType -// @param resumeRecordFile 断点续传的已上传的部分信息记录文件 -// @param progressCallback(uploadBytes, totalBytes) 上传进度回调 -function PutExtra(fname, params, mimeType, resumeRecordFile, progressCallback) { - this.fname = fname || ''; - this.params = params || {}; - this.mimeType = mimeType || null; - this.resumeRecordFile = resumeRecordFile || null; - this.progressCallback = progressCallback || null; +/** + * @callback reqCallback + * + * @param {Error} err + * @param {Object} ret + * @param {http.IncomingMessage} info + */ + +/** + * @callback progressCallback + * + * @param {number} uploadBytes + * @param {number} totalBytes + */ + +/** + * 上传可选参数 + * @param {string} [fname] 请求体中的文件的名称 + * @param {Object} [params] 额外参数设置，参数名称必须以x:开头 + * @param {string | null} [mimeType] 指定文件的mimeType + * @param {string | null} [resumeRecordFile] 断点续传的已上传的部分信息记录文件路径 + * @param {function(number, number):void} [progressCallback] 上传进度回调，回调参数为 (uploadBytes, totalBytes) + * @param {number} [partSize] 分片上传v2必传字段 默认大小为4MB 分片大小范围为1 MB - 1 GB + * @param {'v1' | 'v2'} [version] 分片上传版本 目前支持v1/v2版本 默认v1 + * @param {Object} [metadata] 元数据设置，参数名称必须以 x-qn-meta-${name}: 开头 + */ +function PutExtra ( + fname, + params, + mimeType, + resumeRecordFile, + progressCallback, + partSize, + version, + metadata +) { + this.fname = fname || ''; + this.params = params || {}; + this.mimeType = mimeType || null; + this.resumeRecordFile = resumeRecordFile || null; + this.progressCallback = progressCallback || null; + this.partSize = partSize || conf.BLOCK_SIZE; + this.version = version || 'v1'; + this.metadata = metadata || {}; } -ResumeUploader.prototype.putStream = function(uploadToken, key, rsStream, - rsStreamLen, putExtra, callbackFunc) { - putExtra = putExtra || new PutExtra(); - if (!putExtra.mimeType) { - putExtra.mimeType = 'application/octet-stream'; - } - - if (!putExtra.fname) { - putExtra.fname = key ? key : '?'; - } - - rsStream.on("error", function(err) { - //callbackFunc - callbackFunc(err, null, null); - return; - }); - - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - var accessKey = util.getAKFromUptoken(uploadToken); - var bucket = util.getBucketFromUptoken(uploadToken); - if (useCache) { - putReq(this.config, uploadToken, key, rsStream, rsStreamLen, putExtra, - callbackFunc); - } else { - zone.getZoneInfo(accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - - //req - putReq(that.config, uploadToken, key, rsStream, rsStreamLen, +/** + * @typedef UploadResult + * @property {any} data + * @property {http.IncomingMessage} resp + */ + +/** + * @param {string} uploadToken + * @param {string | null} key + * @param {stream.Readable} rsStream + * @param {number} rsStreamLen + * @param {PutExtra} putExtra + * @param {reqCallback} callbackFunc + * @return {Promise} + */ +ResumeUploader.prototype.putStream = function ( + uploadToken, + key, + rsStream, + rsStreamLen, + putExtra, + callbackFunc +) { + const preferScheme = this.config.useHttpsDomain ? 'https' : 'http'; + const isValidCallback = typeof callbackFunc === 'function'; + + putExtra = getDefaultPutExtra( putExtra, - callbackFunc); + { + key + } + ); + + rsStream.on('error', function (err) { + // callbackFunc + isValidCallback && callbackFunc(err, null, null); + destroy(rsStream); }); - } + + return prepareRegionsProvider({ + config: this.config, + bucketName: util.getBucketFromUptoken(uploadToken), + accessKey: util.getAKFromUptoken(uploadToken) + }) + .then(regionsProvider => { + const resumeInfo = getResumeRecordInfo(putExtra.resumeRecordFile); + let preferredEndpointsProvider; + if (resumeInfo && Array.isArray(resumeInfo.upDomains)) { + preferredEndpointsProvider = new StaticEndpointsProvider( + resumeInfo.upDomains.map(d => new Endpoint(d, { defaultScheme: preferScheme })) + ); + } + return doWorkWithRetry({ + workFn: sendPutReq, + + callbackFunc, + regionsProvider, + // use resume upDomain firstly + preferredEndpointsProvider: preferredEndpointsProvider, + // stream not support retry + retryPolicies: [] + }); + }); + + function sendPutReq (endpoint) { + endpoint = Object.create(endpoint); + endpoint.defaultScheme = preferScheme; + return new Promise(resolve => { + putReq( + endpoint, + uploadToken, + key, + rsStream, + rsStreamLen, + putExtra, + (err, ret, info) => resolve({ err, ret, info })); + }); + } +}; + +/** + * @param {string} resumeRecordFilePath + * @returns {undefined | Object.} + */ +function getResumeRecordInfo (resumeRecordFilePath) { + // get resume record info + let result; + // read resumeRecordFile + if (resumeRecordFilePath) { + try { + const resumeRecords = fs.readFileSync(resumeRecordFilePath).toString(); + result = JSON.parse(resumeRecords); + } catch (e) { + e.code !== 'ENOENT' && console.error(e); + } + } + return result; } -function putReq(config, uploadToken, key, rsStream, rsStreamLen, putExtra, - callbackFunc) { - //set up hosts order - var upHosts = []; +/** + * @param {Endpoint} upEndpoint + * @param {string} uploadToken + * @param {string | null} key + * @param {ReadableStream} rsStream + * @param {number} rsStreamLen + * @param {PutExtra} putExtra + * @param {reqCallback} callbackFunc + */ +function putReq ( + upEndpoint, + uploadToken, + key, + rsStream, + rsStreamLen, + putExtra, + callbackFunc +) { + // make block stream + const blkStream = rsStream.pipe(new BlockStream({ + size: putExtra.partSize, + zeroPadding: false + })); + + // get resume record info + const blkputRets = getResumeRecordInfo(putExtra.resumeRecordFile); + const totalBlockNum = Math.ceil(rsStreamLen / putExtra.partSize); - if (config.useCdnDomain) { - if (config.zone.cdnUpHosts) { - config.zone.cdnUpHosts.forEach(function(host) { - upHosts.push(host); - }); + // select upload version + /** + * @type {function(SourceOptions, UploadOptions, reqCallback)} + */ + let doPutReq; + if (putExtra.version === 'v1') { + doPutReq = putReqV1; + } else if (putExtra.version === 'v2') { + doPutReq = putReqV2; + } else { + throw new Error('part upload version number error'); } - config.zone.srcUpHosts.forEach(function(host) { - upHosts.push(host); - }); - } else { - config.zone.srcUpHosts.forEach(function(host) { - upHosts.push(host); + + // upload parts + doPutReq( + { + blkputRets, + rsStream, + rsStreamLen, + blkStream, + totalBlockNum + }, + { + upEndpoint, + uploadToken, + key, + putExtra + }, + function (err, ret, info) { + if (info.statusCode === 200 && putExtra.resumeRecordFile) { + try { + fs.unlinkSync(putExtra.resumeRecordFile); + } catch (_e) { + // ignore + } + } + callbackFunc(err, ret, info); + } + ); +} + +/** + * @typedef SourceOptions + * @property { Object. | undefined } blkputRets + * @property { ReadableStream } rsStream + * @property { BlockStream } blkStream + * @property { number } rsStreamLen + * @property { number } totalBlockNum + */ + +/** + * @typedef UploadOptions + * @property { string | null } key + * @property { Endpoint } upEndpoint + * @property { string } uploadToken + * @property { PutExtra } putExtra + */ + +/** + * @param {SourceOptions} sourceOptions + * @param {UploadOptions} uploadOptions + * @param {reqCallback} callbackFunc + * @returns { Promise } + */ +function putReqV1 (sourceOptions, uploadOptions, callbackFunc) { + const { + rsStream, + blkStream, + rsStreamLen, + totalBlockNum + } = sourceOptions; + let blkputRets = sourceOptions.blkputRets; + const { + upEndpoint, + key, + uploadToken, + putExtra + } = uploadOptions; + + // initial state + const finishedCtxList = []; + const finishedBlkPutRets = { + upDomains: [], + parts: [] + }; + // backward compatibility with ≤ 7.9.0 + if (Array.isArray(blkputRets)) { + blkputRets = { + upDomains: [], + parts: [] + }; + } + if (blkputRets && Array.isArray(blkputRets.upDomains)) { + finishedBlkPutRets.upDomains = blkputRets.upDomains; + } + finishedBlkPutRets.upDomains.push(upEndpoint.host); + + // upload parts + const upDomains = upEndpoint.getValue(); + let readLen = 0; + let curBlock = 0; + let isSent = false; + blkStream.on('data', function (chunk) { + readLen += chunk.length; + let needUploadBlk = true; + // check uploaded parts + if ( + blkputRets && + blkputRets.parts && + blkputRets.parts.length > 0 && + blkputRets.parts[curBlock] + ) { + const blkputRet = blkputRets.parts[curBlock]; + let expiredAt = blkputRet.expired_at; + // make sure the ctx at least has one day expiration + expiredAt += 3600 * 24; + if (!util.isTimestampExpired(expiredAt)) { + needUploadBlk = false; + finishedCtxList.push(blkputRet.ctx); + finishedBlkPutRets.parts.push(blkputRet); + } + } + + curBlock += 1; // set current block + if (needUploadBlk) { + blkStream.pause(); + mkblkReq( + upDomains, + uploadToken, + chunk, + function ( + respErr, + respBody, + respInfo + ) { + const bodyCrc32 = parseInt('0x' + getCrc32(chunk)); + if (respInfo.statusCode !== 200 || respBody.crc32 !== bodyCrc32) { + callbackFunc(respErr, respBody, respInfo); + destroy(rsStream); + } else { + const blkputRet = respBody; + finishedCtxList.push(blkputRet.ctx); + finishedBlkPutRets.parts.push(blkputRet); + if (putExtra.resumeRecordFile) { + const contents = JSON.stringify(finishedBlkPutRets); + fs.writeFileSync(putExtra.resumeRecordFile, contents, { + encoding: 'utf-8' + }); + } + if (putExtra.progressCallback) { + putExtra.progressCallback(readLen, rsStreamLen); + } + blkStream.resume(); + if (finishedCtxList.length === totalBlockNum) { + mkfileReq(upDomains, uploadToken, rsStreamLen, finishedCtxList, key, putExtra, callbackFunc); + isSent = true; + } + } + }); + } }); - config.zone.cdnUpHosts.forEach(function(host) { - upHosts.push(host); + + blkStream.on('end', function () { + if (!isSent && rsStreamLen === 0) { + mkfileReq(upDomains, uploadToken, rsStreamLen, finishedCtxList, key, putExtra, callbackFunc); + } + destroy(rsStream); }); - } - - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var upDomain = scheme + upHosts[0]; - // block upload - - var fileSize = rsStreamLen; - //console.log("file size:" + fileSize); - var blockCnt = fileSize / conf.BLOCK_SIZE - var totalBlockNum = (fileSize % conf.BLOCK_SIZE == 0) ? blockCnt : (blockCnt + - 1); - var finishedBlock = 0; - var curBlock = 0; - var readLen = 0; - var readBuffers = []; - var finishedCtxList = []; - var finishedBlkPutRets = []; - //read resumeRecordFile - if (putExtra.resumeRecordFile) { - try { - var resumeRecords = fs.readFileSync(putExtra.resumeRecordFile).toString(); - var blkputRets = JSON.parse(resumeRecords); - - for (var index = 0; index < blkputRets.length; index++) { - //check ctx expired or not - var blkputRet = blkputRets[index]; - var expiredAt = blkputRet.expired_at; - //make sure the ctx at least has one day expiration - expiredAt += 3600 * 24; - if (util.isTimestampExpired(expiredAt)) { - //discard these ctxs - break; +} + +/** + * @param {SourceOptions} sourceOptions + * @param {UploadOptions} uploadOptions + * @param {reqCallback} callbackFunc + * @returns { Promise } + */ +function putReqV2 (sourceOptions, uploadOptions, callbackFunc) { + const { + blkputRets, + blkStream, + totalBlockNum, + rsStreamLen, + rsStream + } = sourceOptions; + const { + upEndpoint, + uploadToken, + key, + putExtra + } = uploadOptions; + + // try resume upload blocks + let finishedBlock = 0; + const finishedEtags = { + upDomains: [], + etags: [], + uploadId: '', + expiredAt: 0 + }; + if (blkputRets && Array.isArray(blkputRets.upDomains)) { + // check etag expired or not + const expiredAt = blkputRets.expiredAt; + const timeNow = Date.now() / 1000; + if (expiredAt > timeNow && blkputRets.uploadId) { + finishedEtags.upDomains = blkputRets.upDomains; + finishedEtags.etags = blkputRets.etags; + finishedEtags.uploadId = blkputRets.uploadId; + finishedEtags.expiredAt = blkputRets.expiredAt; + finishedBlock = finishedEtags.etags.length; } + } + finishedEtags.upDomains.push(upEndpoint.host); - finishedBlock += 1; - finishedCtxList.push(blkputRet.ctx); - } - } catch (e) {} - } - - var isEnd = rsStream._readableState.ended; - var isSent = false; - - //check when to mkblk - rsStream.on('data', function(chunk) { - readLen += chunk.length; - readBuffers.push(chunk); - - if (readLen % conf.BLOCK_SIZE == 0 || readLen == fileSize) { - //console.log(readLen); - var readData = Buffer.concat(readBuffers); - readBuffers = []; //reset read buffer - curBlock += 1; //set current block - if (curBlock > finishedBlock) { - rsStream.pause(); - mkblkReq(upDomain, uploadToken, readData, function(respErr, - respBody, - respInfo) { - var bodyCrc32 = parseInt("0x" + getCrc32(readData)); - if (respInfo.statusCode != 200 || respBody.crc32 != bodyCrc32) { - callbackFunc(respErr, respBody, respInfo); + const upDomain = upEndpoint.getValue(); + const bucket = util.getBucketFromUptoken(uploadToken); + const encodedObjectName = key ? util.urlsafeBase64Encode(key) : '~'; + if (finishedEtags.uploadId) { + if (finishedBlock === totalBlockNum) { + completeParts(upDomain, bucket, encodedObjectName, uploadToken, finishedEtags, + putExtra, callbackFunc); return; - } else { - finishedBlock += 1; - var blkputRet = respBody; - finishedCtxList.push(blkputRet.ctx); - finishedBlkPutRets.push(blkputRet); - if (putExtra.progressCallback) { - putExtra.progressCallback(readLen, fileSize); - } - if (putExtra.resumeRecordFile) { - var contents = JSON.stringify(finishedBlkPutRets); - console.log("write resume record " + putExtra.resumeRecordFile) - fs.writeFileSync(putExtra.resumeRecordFile, contents, { - encoding: 'utf-8' - }); - } + } + // if it has resumeRecordFile + resumeUploadV2(uploadToken, bucket, encodedObjectName, upDomain, blkStream, + finishedEtags, finishedBlock, totalBlockNum, putExtra, rsStreamLen, rsStream, callbackFunc); + } else { + // init a new uploadId for next step + initReq(uploadToken, bucket, encodedObjectName, upDomain, blkStream, + finishedEtags, finishedBlock, totalBlockNum, putExtra, rsStreamLen, rsStream, callbackFunc); + } +} + +/** + * @param {string} upDomain + * @param {string} uploadToken + * @param {Buffer | string} blkData + * @param {reqCallback} callbackFunc + */ +function mkblkReq (upDomain, uploadToken, blkData, callbackFunc) { + const requestURI = upDomain + '/mkblk/' + blkData.length; + const auth = 'UpToken ' + uploadToken; + const headers = { + Authorization: auth, + 'Content-Type': 'application/octet-stream' + }; + rpc.post(requestURI, blkData, headers, callbackFunc); +} - rsStream.resume(); - if (isEnd || finishedCtxList.length === Math.floor(totalBlockNum)) { - mkfileReq(upDomain, uploadToken, fileSize, finishedCtxList, key, putExtra, callbackFunc); - isSent = true; +/** + * @param {string} upDomain + * @param {string} uploadToken + * @param {number} fileSize + * @param {string[]} ctxList + * @param {string | null} key + * @param putExtra + * @param callbackFunc + */ +function mkfileReq ( + upDomain, + uploadToken, + fileSize, + ctxList, + key, + putExtra, + callbackFunc +) { + let requestURI = upDomain + '/mkfile/' + fileSize; + if (key) { + requestURI += '/key/' + util.urlsafeBase64Encode(key); + } + if (putExtra.mimeType) { + requestURI += '/mimeType/' + util.urlsafeBase64Encode(putExtra.mimeType); + } + if (putExtra.fname) { + requestURI += '/fname/' + util.urlsafeBase64Encode(putExtra.fname); + } + if (putExtra.params) { + // putExtra params + for (const k in putExtra.params) { + if (k.startsWith('x:') && putExtra.params[k]) { + requestURI += '/' + k + '/' + util.urlsafeBase64Encode(putExtra.params[ + k].toString()); } - } - }); - } + } } - }); - rsStream.on('end', function () { - // 0B file won't trigger 'data' event - if (!isSent && rsStreamLen === 0) { - mkfileReq(upDomain, uploadToken, fileSize, finishedCtxList, key, putExtra, callbackFunc) + // putExtra metadata + if (putExtra.metadata) { + for (const metadataKey in putExtra.metadata) { + if (metadataKey.startsWith('x-qn-meta-') && putExtra.metadata[metadataKey]) { + requestURI += + '/' + metadataKey + '/' + + util.urlsafeBase64Encode(putExtra.metadata[metadataKey].toString()); + } + } } - }) + + const auth = 'UpToken ' + uploadToken; + const headers = { + Authorization: auth, + 'Content-Type': 'application/octet-stream' + }; + const postBody = ctxList.join(','); + rpc.post(requestURI, postBody, headers, callbackFunc); } -function mkblkReq(upDomain, uploadToken, blkData, callbackFunc) { - //console.log("mkblk"); - var requestURI = upDomain + "/mkblk/" + blkData.length; - var auth = 'UpToken ' + uploadToken; - var headers = { - 'Authorization': auth, - 'Content-Type': 'application/octet-stream' - } - rpc.post(requestURI, blkData, headers, callbackFunc); +/** + * @typedef FinishedEtags + * @property {{etag: string, partNumber: number}[]}etags + * @property {string} uploadId + * @property {number} expiredAt + */ + +/** + * @param {string} uploadToken + * @param {string} bucket + * @param {string} encodedObjectName + * @param {string} upDomain + * @param {BlockStream} blkStream + * @param {FinishedEtags} finishedEtags + * @param {number} finishedBlock + * @param {number} totalBlockNum + * @param {PutExtra} putExtra + * @param {number} rsStreamLen + * @param {stream.Readable} rsStream + * @param {reqCallback} callbackFunc + */ +function initReq ( + uploadToken, + bucket, + encodedObjectName, + upDomain, + blkStream, + finishedEtags, + finishedBlock, + totalBlockNum, + putExtra, + rsStreamLen, + rsStream, + callbackFunc +) { + const requestUrl = upDomain + '/buckets/' + bucket + '/objects/' + encodedObjectName + '/uploads'; + const headers = { + Authorization: 'UpToken ' + uploadToken, + 'Content-Type': 'application/json' + }; + rpc.post(requestUrl, '', headers, function (err, ret, info) { + if (info.statusCode !== 200) { + callbackFunc(err, ret, info); + return; + } + finishedEtags.expiredAt = ret.expireAt; + finishedEtags.uploadId = ret.uploadId; + resumeUploadV2(uploadToken, bucket, encodedObjectName, upDomain, blkStream, + finishedEtags, finishedBlock, totalBlockNum, putExtra, rsStreamLen, rsStream, callbackFunc); + }); } -function mkfileReq(upDomain, uploadToken, fileSize, ctxList, key, putExtra, - callbackFunc) { - //console.log("mkfile"); - var requestURI = upDomain + "/mkfile/" + fileSize; - if (key) { - requestURI += "/key/" + util.urlsafeBase64Encode(key); - } - if (putExtra.mimeType) { - requestURI += "/mimeType/" + util.urlsafeBase64Encode(putExtra.mimeType); - } - if (putExtra.fname) { - requestURI += "/fname/" + util.urlsafeBase64Encode(putExtra.fname); - } - if (putExtra.params) { - //putExtra params - for (var k in putExtra.params) { - if (k.startsWith("x:") && putExtra.params[k]) { - requestURI += "/" + k + "/" + util.urlsafeBase64Encode(putExtra.params[ - k].toString()); - } - } - } - var auth = 'UpToken ' + uploadToken; - var headers = { - 'Authorization': auth, - 'Content-Type': 'application/octet-stream' - } - var postBody = ctxList.join(","); - rpc.post(requestURI, postBody, headers, function(err, ret, info) { - if (info.statusCode == 200 || info.statusCode == 701 || - info.statusCode == 401) { - if (putExtra.resumeRecordFile) { - fs.unlinkSync(putExtra.resumeRecordFile); - } - } - callbackFunc(err, ret, info); - }); +/** + * @param {string} uploadToken + * @param {string} bucket + * @param {string} encodedObjectName + * @param {string} upDomain + * @param {BlockStream} blkStream + * @param {FinishedEtags} finishedEtags + * @param {number} finishedBlock + * @param {number} totalBlockNum + * @param {PutExtra} putExtra + * @param {number} rsStreamLen + * @param {stream.Readable} rsStream + * @param {reqCallback} callbackFunc + */ +function resumeUploadV2 ( + uploadToken, + bucket, + encodedObjectName, + upDomain, + blkStream, + finishedEtags, + finishedBlock, + totalBlockNum, + putExtra, + rsStreamLen, + rsStream, + callbackFunc +) { + let isSent = false; + let readLen = 0; + let curBlock = 0; + blkStream.on('data', function (chunk) { + let partNumber = 0; + readLen += chunk.length; + curBlock += 1; // set current block + if (curBlock > finishedBlock) { + blkStream.pause(); + partNumber = finishedBlock + 1; + const bodyMd5 = util.getMd5(chunk); + uploadPart(bucket, upDomain, uploadToken, encodedObjectName, chunk, finishedEtags.uploadId, partNumber, putExtra, + function (respErr, respBody, respInfo) { + if (respInfo.statusCode !== 200 || respBody.md5 !== bodyMd5) { + callbackFunc(respErr, respBody, respInfo); + destroy(rsStream); + } else { + finishedBlock += 1; + const blockStatus = { + etag: respBody.etag, + partNumber: partNumber + }; + finishedEtags.etags.push(blockStatus); + if (putExtra.resumeRecordFile) { + const contents = JSON.stringify(finishedEtags); + fs.writeFileSync(putExtra.resumeRecordFile, contents, { + encoding: 'utf-8' + }); + } + if (putExtra.progressCallback) { + putExtra.progressCallback(readLen, rsStreamLen); + } + blkStream.resume(); + if (finishedEtags.etags.length === totalBlockNum) { + completeParts(upDomain, bucket, encodedObjectName, uploadToken, finishedEtags, + putExtra, callbackFunc); + isSent = true; + } + } + }); + } + }); + + blkStream.on('end', function () { + if (!isSent && rsStreamLen === 0) { + completeParts(upDomain, bucket, encodedObjectName, uploadToken, finishedEtags, + putExtra, callbackFunc); + } + destroy(rsStream); + }); } -ResumeUploader.prototype.putFile = function(uploadToken, key, localFile, - putExtra, callbackFunc) { - putExtra = putExtra || new PutExtra(); - var rsStream = fs.createReadStream(localFile); - var rsStreamLen = fs.statSync(localFile).size; - if (!putExtra.mimeType) { - putExtra.mimeType = mime.lookup(localFile); - } - - if (!putExtra.fname) { - putExtra.fname = path.basename(localFile); - } - - return this.putStream(uploadToken, key, rsStream, rsStreamLen, putExtra, - callbackFunc); +/** + * @param {string} bucket + * @param {string} upDomain + * @param {string} uploadToken + * @param {string} encodedObjectName + * @param {Buffer | string} chunk + * @param {string} uploadId + * @param {number} partNumber + * @param {PutExtra} putExtra + * @param {reqCallback} callbackFunc + */ +function uploadPart (bucket, upDomain, uploadToken, encodedObjectName, chunk, uploadId, partNumber, putExtra, callbackFunc) { + const headers = { + Authorization: 'UpToken ' + uploadToken, + 'Content-Type': 'application/octet-stream', + 'Content-MD5': util.getMd5(chunk) + }; + const requestUrl = upDomain + '/buckets/' + bucket + '/objects/' + encodedObjectName + '/uploads/' + uploadId + + '/' + partNumber.toString(); + rpc.put(requestUrl, chunk, headers, callbackFunc); } -ResumeUploader.prototype.putFileWithoutKey = function(uploadToken, localFile, - putExtra, callbackFunc) { - return this.putFile(uploadToken, null, localFile, putExtra, callbackFunc); +/** + * @param {string} upDomain + * @param {string} bucket + * @param {string} encodedObjectName + * @param {string} uploadToken + * @param {FinishedEtags} finishedEtags + * @param {PutExtra} putExtra + * @param {reqCallback} callbackFunc + */ +function completeParts ( + upDomain, + bucket, + encodedObjectName, + uploadToken, + finishedEtags, + putExtra, + callbackFunc +) { + const headers = { + Authorization: 'UpToken ' + uploadToken, + 'Content-Type': 'application/json' + }; + const sortedParts = finishedEtags.etags.sort(function (a, b) { + return a.partNumber - b.partNumber; + }); + const body = { + fname: putExtra.fname, + mimeType: putExtra.mimeType, + customVars: putExtra.params, + metadata: putExtra.metadata, + parts: sortedParts + }; + const requestUrl = upDomain + '/buckets/' + bucket + '/objects/' + encodedObjectName + '/uploads/' + finishedEtags.uploadId; + const requestBody = JSON.stringify(body); + rpc.post( + requestUrl, + requestBody, + headers, + callbackFunc + ); +} + +/** + * @param {string} uploadToken + * @param {string | null} key + * @param {string} localFile + * @param {PutExtra} putExtra + * @param {reqCallback} callbackFunc + * @returns {Promise} + */ +ResumeUploader.prototype.putFile = function ( + uploadToken, + key, + localFile, + putExtra, + callbackFunc +) { + const preferScheme = this.config.useHttpsDomain ? 'https' : 'http'; + + // PutExtra + putExtra = putExtra || new PutExtra(); + if (!putExtra.mimeType) { + putExtra.mimeType = mime.getType(localFile); + } + + if (!putExtra.fname) { + putExtra.fname = path.basename(localFile); + } + + putExtra = getDefaultPutExtra( + putExtra, + { + key + } + ); + + // regions + return prepareRegionsProvider({ + config: this.config, + bucketName: util.getBucketFromUptoken(uploadToken), + accessKey: util.getAKFromUptoken(uploadToken) + }) + .then(regionsProvider => { + const resumeInfo = getResumeRecordInfo(putExtra.resumeRecordFile); + let preferredEndpointsProvider; + if (resumeInfo && Array.isArray(resumeInfo.upDomains)) { + preferredEndpointsProvider = new StaticEndpointsProvider( + resumeInfo.upDomains.map(d => new Endpoint(d, { defaultScheme: preferScheme })) + ); + } + return doWorkWithRetry({ + workFn: sendPutReq, + + callbackFunc, + regionsProvider, + uploadApiVersion: putExtra.version, + // use resume upDomain firstly + preferredEndpointsProvider: preferredEndpointsProvider, + resumeRecordFilePath: putExtra.resumeRecordFile, + retryPolicies: this.retryPolicies + }); + }); + + function sendPutReq (endpoint) { + endpoint = Object.create(endpoint); + endpoint.defaultScheme = preferScheme; + const rsStream = fs.createReadStream(localFile, { + highWaterMark: conf.BLOCK_SIZE + }); + const rsStreamLen = fs.statSync(localFile).size; + return new Promise((resolve) => { + putReq( + endpoint, + uploadToken, + key, + rsStream, + rsStreamLen, + putExtra, + (err, ret, info) => { + destroy(rsStream); + resolve({ err, ret, info }); + } + ); + }); + } +}; + +/** + * @param {string} uploadToken + * @param {string} localFile + * @param {PutExtra} putExtra + * @param {reqCallback} callbackFunc + * @returns {Promise} + */ +ResumeUploader.prototype.putFileWithoutKey = function ( + uploadToken, + localFile, + putExtra, + callbackFunc +) { + return this.putFile(uploadToken, null, localFile, putExtra, callbackFunc); +}; + +/** + * @param {PutExtra} putExtra + * @param {Object} options + * @param {string | null} [options.key] + * @return {PutExtra} + */ +function getDefaultPutExtra (putExtra, options) { + options = options || {}; + + putExtra = putExtra || new PutExtra(); + if (!putExtra.mimeType) { + putExtra.mimeType = 'application/octet-stream'; + } + + if (!putExtra.fname) { + putExtra.fname = options.key || '?'; + } + + if (!putExtra.version) { + putExtra.version = 'v1'; + } + + return putExtra; } diff --git a/qiniu/storage/rs.js b/qiniu/storage/rs.js index 7aad7295..3d3939ba 100644 --- a/qiniu/storage/rs.js +++ b/qiniu/storage/rs.js @@ -1,20 +1,16 @@ -const url = require('url'); -const crypto = require('crypto'); -const formstream = require('formstream'); const querystring = require('querystring'); const encodeUrl = require('encodeurl'); const rpc = require('../rpc'); const conf = require('../conf'); const digest = require('../auth/digest'); const util = require('../util'); -const zone = require('../zone'); exports.BucketManager = BucketManager; exports.PutPolicy = PutPolicy; function BucketManager(mac, config) { - this.mac = mac || new digest.Mac(); - this.config = config || new conf.Config(); + this.mac = mac || new digest.Mac(); + this.config = config || new conf.Config(); } // 获取资源信息 @@ -22,44 +18,28 @@ function BucketManager(mac, config) { // @param bucket 空间名称 // @param key 文件名称 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.stat = function(bucket, key, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - statReq(this.mac, this.config, bucket, key, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - statReq(that.mac, that.config, bucket, key, callbackFunc); +BucketManager.prototype.stat = function (bucket, key, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + statReq(ctx.mac, ctx.config, bucket, key, callbackFunc); }); - } -} - -function statReq(mac, config, bucket, key, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var statOp = exports.statOp(bucket, key); - var requestURI = scheme + config.zone.rsHost + statOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); +}; + +function statReq (mac, config, bucket, key, callbackFunc) { + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var statOp = exports.statOp(bucket, key); + var requestURI = scheme + config.zone.rsHost + statOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } // 修改文件的类型 @@ -68,99 +48,64 @@ function statReq(mac, config, bucket, key, callbackFunc) { // @param key 文件名称 // @param newMime 新文件类型 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.changeMime = function(bucket, key, newMime, - callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - changeMimeReq(this.mac, this.config, bucket, key, newMime, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - changeMimeReq(that.mac, that.config, bucket, key, newMime, - callbackFunc); +BucketManager.prototype.changeMime = function (bucket, key, newMime, + callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + changeMimeReq(ctx.mac, ctx.config, bucket, key, newMime, callbackFunc); }); - } -} +}; function changeMimeReq(mac, config, bucket, key, newMime, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var changeMimeOp = exports.changeMimeOp(bucket, key, newMime); - var requestURI = scheme + config.zone.rsHost + changeMimeOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var changeMimeOp = exports.changeMimeOp(bucket, key, newMime); + var requestURI = scheme + config.zone.rsHost + changeMimeOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } - // 修改文件返回的Headers内容 // @link TODO // @param bucket 空间名称 // @param key 文件名称 // @param headers 需要修改的headers // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.changeHeaders = function(bucket, key, headers, - callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - changeHeadersReq(this.mac, this.config, bucket, key, headers, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - changeHeadersReq(that.mac, that.config, bucket, key, headers, - callbackFunc); +BucketManager.prototype.changeHeaders = function (bucket, key, headers, + callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + changeHeadersReq(ctx.mac, ctx.config, bucket, key, headers, callbackFunc); }); - } -} +}; function changeHeadersReq(mac, config, bucket, key, headers, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var changeHeadersOp = exports.changeHeadersOp(bucket, key, headers); - var requestURI = scheme + config.zone.rsHost + changeHeadersOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var changeHeadersOp = exports.changeHeadersOp(bucket, key, headers); + var requestURI = scheme + config.zone.rsHost + changeHeadersOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } // 移动或重命名文件，当bucketSrc==bucketDest相同的时候，就是重命名文件操作 -// @link https://developer.qiniu.com/kodo/api/1257/delete +// @link https://developer.qiniu.com/kodo/1288/move // @param srcBucket 源空间名称 // @param srcKey 源文件名称 // @param destBucket 目标空间名称 @@ -168,48 +113,31 @@ function changeHeadersReq(mac, config, bucket, key, headers, callbackFunc) { // @param options 可选参数 // force 强制覆盖 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.move = function(srcBucket, srcKey, destBucket, destKey, - options, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - moveReq(this.mac, this.config, srcBucket, srcKey, destBucket, destKey, - options, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, srcBucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - moveReq(that.mac, that.config, srcBucket, srcKey, destBucket, - destKey, options, callbackFunc); +BucketManager.prototype.move = function (srcBucket, srcKey, destBucket, destKey, + options, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, srcBucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + moveReq(ctx.mac, ctx.config, srcBucket, srcKey, destBucket, destKey, + options, callbackFunc); }); - } -} +}; function moveReq(mac, config, srcBucket, srcKey, destBucket, destKey, - options, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var moveOp = exports.moveOp(srcBucket, srcKey, destBucket, destKey, options); - var requestURI = scheme + config.zone.rsHost + moveOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + options, callbackFunc) { + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var moveOp = exports.moveOp(srcBucket, srcKey, destBucket, destKey, options); + var requestURI = scheme + config.zone.rsHost + moveOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } // 复制一个文件 @@ -221,49 +149,32 @@ function moveReq(mac, config, srcBucket, srcKey, destBucket, destKey, // @param options 可选参数 // force 强制覆盖 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.copy = function(srcBucket, srcKey, destBucket, destKey, - options, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - copyReq(this.mac, this.config, srcBucket, srcKey, destBucket, destKey, - options, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, srcBucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - copyReq(that.mac, that.config, srcBucket, srcKey, destBucket, - destKey, options, callbackFunc); +BucketManager.prototype.copy = function (srcBucket, srcKey, destBucket, destKey, + options, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, srcBucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + copyReq(ctx.mac, ctx.config, srcBucket, srcKey, destBucket, destKey, + options, callbackFunc); }); - } -} +}; function copyReq(mac, config, srcBucket, srcKey, destBucket, destKey, - options, callbackFunc) { - options = options || {}; - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var copyOp = exports.copyOp(srcBucket, srcKey, destBucket, destKey, options); - var requestURI = scheme + config.zone.rsHost + copyOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + options, callbackFunc) { + options = options || {}; + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var copyOp = exports.copyOp(srcBucket, srcKey, destBucket, destKey, options); + var requestURI = scheme + config.zone.rsHost + copyOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } // 删除资源 @@ -271,93 +182,104 @@ function copyReq(mac, config, srcBucket, srcKey, destBucket, destKey, // @param bucket 空间名称 // @param key 文件名称 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.delete = function(bucket, key, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - deleteReq(this.mac, this.config, bucket, key, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - deleteReq(that.mac, that.config, bucket, key, callbackFunc); +BucketManager.prototype.delete = function (bucket, key, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + deleteReq(ctx.mac, ctx.config, bucket, key, callbackFunc); }); - } -} +}; function deleteReq(mac, config, bucket, key, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var deleteOp = exports.deleteOp(bucket, key); - var requestURI = scheme + config.zone.rsHost + deleteOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var deleteOp = exports.deleteOp(bucket, key); + var requestURI = scheme + config.zone.rsHost + deleteOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } - // 更新文件的生命周期 // @link https://developer.qiniu.com/kodo/api/1732/update-file-lifecycle // @param bucket 空间名称 // @param key 文件名称 // @param days 有效期天数 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.deleteAfterDays = function(bucket, key, days, - callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - deleteAfterDaysReq(this.mac, this.config, bucket, key, days, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - deleteAfterDaysReq(that.mac, that.config, bucket, key, days, - callbackFunc); +BucketManager.prototype.deleteAfterDays = function (bucket, key, days, + callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + deleteAfterDaysReq(ctx.mac, ctx.config, bucket, key, days, callbackFunc); }); - } -} +}; function deleteAfterDaysReq(mac, config, bucket, key, days, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var deleteAfterDaysOp = exports.deleteAfterDaysOp(bucket, key, days); - var requestURI = scheme + config.zone.rsHost + deleteAfterDaysOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var deleteAfterDaysOp = exports.deleteAfterDaysOp(bucket, key, days); + var requestURI = scheme + config.zone.rsHost + deleteAfterDaysOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); +} + +/** + * @param { string } bucket - 空间名称 + * @param { string } key - 文件名称 + * @param { Object } options - 配置项 + * @param { number } options.toIaAfterDays - 多少天后将文件转为低频存储，设置为 -1 表示取消已设置的转低频存储的生命周期规则， 0 表示不修改转低频生命周期规则。 + * @param { number } options.toArchiveAfterDays - 多少天后将文件转为归档存储，设置为 -1 表示取消已设置的转归档存储的生命周期规则， 0 表示不修改转归档生命周期规则。 + * @param { number } options.toArchiveIRAfterDays - 多少天后将文件转为归档直读存储，设置为 -1 表示取消已设置的转归档直读存储的生命周期规则， 0 表示不修改转归档直读生命周期规则。 + * @param { number } options.toDeepArchiveAfterDays - 多少天后将文件转为深度归档存储，设置为 -1 表示取消已设置的转深度归档存储的生命周期规则， 0 表示不修改转深度归档生命周期规则。 + * @param { number } options.deleteAfterDays - 多少天后将文件删除，设置为 -1 表示取消已设置的删除存储的生命周期规则， 0 表示不修改删除存储的生命周期规则。 + * @param { Object } options.cond - 匹配条件，只有条件匹配才会设置成功 + * @param { string } options.cond.hash + * @param { string } options.cond.mime + * @param { number } options.cond.fsize + * @param { number } options.cond.putTime + * @param { function } callbackFunc - 回调函数 + */ +BucketManager.prototype.setObjectLifeCycle = function ( + bucket, + key, + options, + callbackFunc +) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + setObjectLifecycleReq(ctx.mac, ctx.config, bucket, key, options, callbackFunc); + }); +}; + +function setObjectLifecycleReq (mac, config, bucket, key, options, callbackFunc) { + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const setObjectLifecycleOp = exports.setObjectLifecycleOp(bucket, key, options); + const requestUrl = scheme + config.zone.rsHost + setObjectLifecycleOp; + rpc.postWithOptions( + requestUrl, + null, + { + mac + }, + callbackFunc + ); } // 抓取资源 @@ -366,46 +288,30 @@ function deleteAfterDaysReq(mac, config, bucket, key, days, callbackFunc) { // @param bucket 空间名称 // @param key 文件名称 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.fetch = function(resUrl, bucket, key, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - fetchReq(this.mac, this.config, resUrl, bucket, key, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - fetchReq(that.mac, that.config, resUrl, bucket, key, callbackFunc); +BucketManager.prototype.fetch = function (resUrl, bucket, key, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + fetchReq(ctx.mac, ctx.config, resUrl, bucket, key, callbackFunc); }); - } -} +}; function fetchReq(mac, config, resUrl, bucket, key, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var encodedEntryURI = util.encodedEntry(bucket, key); - var encodedResURL = util.urlsafeBase64Encode(resUrl); - var requestURI = scheme + config.zone.ioHost + '/fetch/' + encodedResURL + - '/to/' + encodedEntryURI; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var encodedEntryURI = util.encodedEntry(bucket, key); + var encodedResURL = util.urlsafeBase64Encode(resUrl); + var requestURI = scheme + config.zone.ioHost + '/fetch/' + encodedResURL + + '/to/' + encodedEntryURI; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } // 更新镜像副本 @@ -413,44 +319,28 @@ function fetchReq(mac, config, resUrl, bucket, key, callbackFunc) { // @param bucket 空间名称 // @param key 文件名称 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.prefetch = function(bucket, key, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - prefetchReq(this.mac, this.config, bucket, key, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - prefetchReq(that.mac, that.config, bucket, key, callbackFunc); +BucketManager.prototype.prefetch = function (bucket, key, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + prefetchReq(ctx.mac, ctx.config, bucket, key, callbackFunc); }); - } -} +}; function prefetchReq(mac, config, bucket, key, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var encodedEntryURI = util.encodedEntry(bucket, key); - var requestURI = scheme + config.zone.ioHost + '/prefetch/' + encodedEntryURI; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var encodedEntryURI = util.encodedEntry(bucket, key); + var requestURI = scheme + config.zone.ioHost + '/prefetch/' + encodedEntryURI; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } // 修改文件的存储类型 @@ -459,230 +349,338 @@ function prefetchReq(mac, config, bucket, key, callbackFunc) { // @param key 文件名称 // @param newType 新文件存储类型 // @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.changeType = function(bucket, key, newType, - callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; - } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - changeTypeReq(this.mac, this.config, bucket, key, newType, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } - - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - changeTypeReq(that.mac, that.config, bucket, key, newType, - callbackFunc); +BucketManager.prototype.changeType = function (bucket, key, newType, + callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + changeTypeReq(ctx.mac, ctx.config, bucket, key, newType, callbackFunc); }); - } -} +}; function changeTypeReq(mac, config, bucket, key, newType, callbackFunc) { - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var changeTypeOp = exports.changeTypeOp(bucket, key, newType); - var requestURI = scheme + config.zone.rsHost + changeTypeOp; - var digest = util.generateAccessToken(mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var changeTypeOp = exports.changeTypeOp(bucket, key, newType); + var requestURI = scheme + config.zone.rsHost + changeTypeOp; + rpc.postWithOptions( + requestURI, + null, + { + mac + }, + callbackFunc + ); } +/** + * 设置空间镜像源 + * @link https://developer.qiniu.com/kodo/3966/bucket-image-source + * @param {string} bucket 空间名称 + * @param {string} srcSiteUrl 镜像源地址 + * @param {string} srcHost 镜像Host + * @param {function(err: error, respBody: object, respInfo: object)} callbackFunc 回调函数 + */ +BucketManager.prototype.image = function (bucket, srcSiteUrl, srcHost, + callbackFunc) { + const encodedSrcSite = util.urlsafeBase64Encode(srcSiteUrl); + const scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + let requestURI = scheme + conf.UC_HOST + '/image/' + bucket + '/from/' + encodedSrcSite; + if (srcHost) { + const encodedHost = util.urlsafeBase64Encode(srcHost); + requestURI += '/host/' + encodedHost; + } + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +/** + * 取消设置空间镜像源 + * @param {string} bucket 空间名称 + * @param {function(err: error, respBody: object, respInfo: object)} callbackFunc 回调函数 + */ +BucketManager.prototype.unimage = function (bucket, callbackFunc) { + const scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + const requestURI = scheme + conf.UC_HOST + '/unimage/' + bucket; + const digest = util.generateAccessTokenV2(this.mac, requestURI, 'POST', 'application/x-www-form-urlencoded'); + rpc.postWithoutForm(requestURI, digest, callbackFunc); +}; + +/** + * 获取指定前缀的文件列表 + * @link https://developer.qiniu.com/kodo/api/1284/list + * + * @param { string } bucket 空间名称 + * @param { Object } options 列举操作的可选参数 + * @param { string } options.prefix 列举的文件前缀 + * @param { string } options.marker 上一次列举返回的位置标记，作为本次列举的起点信息 + * @param { number } options.limit 每次返回的最大列举文件数量 + * @param { string } options.delimiter 指定目录分隔符 + * @param { function } callbackFunc(err, respBody, respInfo) 回调函数 + */ +BucketManager.prototype.listPrefix = function (bucket, options, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + listPrefixReq(ctx.mac, ctx.config, bucket, options, callbackFunc); + }); +}; -// 设置空间镜像源 -// @link https://developer.qiniu.com/kodo/api/1370/mirror -// @param bucket 空间名称 -// @param srcSiteUrl 镜像源地址 -// @param srcHost 镜像Host -// @param callbackFunc(err, respBody, respInfo) 回调函数 -const PU_HOST = "http://pu.qbox.me:10200"; -BucketManager.prototype.image = function(bucket, srcSiteUrl, srcHost, - callbackFunc) { - var encodedSrcSite = util.urlsafeBase64Encode(srcSiteUrl); - var requestURI = PU_HOST + "/image/" + bucket + "/from/" + encodedSrcSite; - if (srcHost) { - var encodedHost = util.urlsafeBase64Encode(srcHost); - requestURI += "/host/" + encodedHost; - } - var digest = util.generateAccessToken(this.mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); -} +function listPrefixReq (mac, config, bucket, options, callbackFunc) { + options = options || {}; + // 必须参数 + const reqParams = { + bucket: bucket + }; -// 取消设置空间镜像源 -// @link https://developer.qiniu.com/kodo/api/1370/mirror -// @param bucket 空间名称 -// @param callbackFunc(err, respBody, respInfo) 回调函数 -BucketManager.prototype.unimage = function(bucket, callbackFunc) { - var requestURI = PU_HOST + "/unimage/" + bucket; - var digest = util.generateAccessToken(this.mac, requestURI, null); - rpc.postWithoutForm(requestURI, digest, callbackFunc); -} + if (options.prefix) { + reqParams.prefix = options.prefix; + } else { + reqParams.prefix = ''; + } -// 获取指定前缀的文件列表 -// @link https://developer.qiniu.com/kodo/api/1284/list -// -// @param bucket 空间名称 -// @param options 列举操作的可选参数 -// prefix 列举的文件前缀 -// marker 上一次列举返回的位置标记，作为本次列举的起点信息 -// limit 每次返回的最大列举文件数量 -// delimiter 指定目录分隔符 -// @param callbackFunc(err, respBody, respInfo) - 回调函数 -BucketManager.prototype.listPrefix = function(bucket, options, callbackFunc) { - var useCache = false; - var that = this; - if (this.config.zone) { - if (this.config.zoneExpire == -1) { - useCache = true; + if (options.limit >= 1 && options.limit <= 1000) { + reqParams.limit = options.limit; } else { - if (!util.isTimestampExpired(this.config.zoneExpire)) { - useCache = true; - } - } - } - - if (useCache) { - listPrefixReq(this.mac, this.config, bucket, options, callbackFunc); - } else { - zone.getZoneInfo(this.mac.accessKey, bucket, function(err, cZoneInfo, - cZoneExpire) { - if (err) { - callbackFunc(err, null, null); - return; - } + reqParams.limit = 1000; + } - //update object - that.config.zone = cZoneInfo; - that.config.zoneExpire = cZoneExpire; - //req - listPrefixReq(that.mac, that.config, bucket, options, callbackFunc); - }); - } -} + if (options.marker) { + reqParams.marker = options.marker; + } else { + reqParams.marker = ''; + } -function listPrefixReq(mac, config, bucket, options, callbackFunc) { - options = options || {}; - //必须参数 - var reqParams = { - bucket: bucket, - }; - - if (options.prefix) { - reqParams.prefix = options.prefix; - } else { - reqParams.prefix = ""; - } - - if (options.limit >= 1 && options.limit <= 1000) { - reqParams.limit = options.limit; - } else { - reqParams.limit = 1000; - } - - if (options.marker) { - reqParams.marker = options.marker; - } else { - reqParams.marker = ""; - } - - if (options.delimiter) { - reqParams.delimiter = options.delimiter; - } else { - reqParams.delimiter = ""; - } - - var scheme = config.useHttpsDomain ? "https://" : "http://"; - var reqSpec = querystring.stringify(reqParams); - var requestURI = scheme + config.zone.rsfHost + '/list?' + reqSpec; - - var auth = util.generateAccessToken(mac, requestURI, null); - rpc.postWithForm(requestURI, null, auth, callbackFunc); -} + if (options.delimiter) { + reqParams.delimiter = options.delimiter; + } else { + reqParams.delimiter = ''; + } -// 批量文件管理请求，支持stat，chgm，chtype，delete，copy，move -BucketManager.prototype.batch = function(operations, callbackFunc) { - var requestURI = conf.RS_HOST + "/batch"; - var reqParams = { - op: operations, - }; - var reqBody = querystring.stringify(reqParams); - var digest = util.generateAccessToken(this.mac, requestURI, reqBody); - rpc.postWithForm(requestURI, reqBody, digest, callbackFunc); + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const reqSpec = querystring.stringify(reqParams); + const requestURI = scheme + config.zone.rsfHost + '/list?' + reqSpec; + + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); } -// 批量操作支持的指令构造器 -exports.statOp = function(bucket, key) { - return "/stat/" + util.encodedEntry(bucket, key); -} +/** + * 获取指定前缀的文件列表 V2 + * + * @deprecated API 可能返回仅包含 marker，不包含 item 或 dir 的项，请使用 {@link listPrefix} + * + * @param bucket 空间名称 + * @param { Object } options 列举操作的可选参数 + * @param { string } options.prefix 列举的文件前缀 + * @param { string } options.marker 上一次列举返回的位置标记，作为本次列举的起点信息 + * @param { number } options.limit 每次返回的最大列举文件数量 + * @param { string } options.delimiter 指定目录分隔符 + * @param { function } callbackFunc(err, respBody, respInfo) 回调函数 + */ +BucketManager.prototype.listPrefixV2 = function (bucket, options, callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + listPrefixReqV2(ctx.mac, ctx.config, bucket, options, callbackFunc); + }); +}; -exports.deleteOp = function(bucket, key) { - return "/delete/" + util.encodedEntry(bucket, key); -} +function listPrefixReqV2 (mac, config, bucket, options, callbackFunc) { + options = options || {}; + // 必须参数 + const reqParams = { + bucket: bucket + }; -exports.deleteAfterDaysOp = function(bucket, key, days) { - var encodedEntryURI = util.encodedEntry(bucket, key); - return '/deleteAfterDays/' + encodedEntryURI + "/" + days; -} + if (options.prefix) { + reqParams.prefix = options.prefix; + } else { + reqParams.prefix = ''; + } -exports.changeMimeOp = function(bucket, key, newMime) { - var encodedEntryURI = util.encodedEntry(bucket, key); - var encodedMime = util.urlsafeBase64Encode(newMime); - return '/chgm/' + encodedEntryURI + '/mime/' + encodedMime; -} + if (options.limit) { + reqParams.limit = Math.min(1000, Math.max(0, options.limit)); + } else { + reqParams.limit = 0; + } -exports.changeHeadersOp = function(bucket, key, headers) { - var encodedEntryURI = util.encodedEntry(bucket, key); - var prefix = 'x-qn-meta-!'; - var path = '/chgm/' + encodedEntryURI; - for (var headerKey in headers) { - var encodedValue = util.urlsafeBase64Encode(headers[headerKey]); - var prefixedHeaderKey = prefix + headerKey; - path += "/" + prefixedHeaderKey + "/" + encodedValue; - } - - return path; -} + if (options.marker) { + reqParams.marker = options.marker; + } else { + reqParams.marker = ''; + } -exports.changeTypeOp = function(bucket, key, newType) { - var encodedEntryURI = util.encodedEntry(bucket, key); - return '/chtype/' + encodedEntryURI + '/type/' + newType; -} + if (options.delimiter) { + reqParams.delimiter = options.delimiter; + } else { + reqParams.delimiter = ''; + } -exports.moveOp = function(srcBucket, srcKey, destBucket, destKey, options) { - options = options || {}; - var encodedEntryURISrc = util.encodedEntry(srcBucket, srcKey); - var encodedEntryURIDest = util.encodedEntry(destBucket, destKey); - var op = "/move/" + encodedEntryURISrc + "/" + encodedEntryURIDest; - if (options.force) { - op += "/force/true"; - } - return op; + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const reqSpec = querystring.stringify(reqParams); + const requestURI = scheme + config.zone.rsfHost + '/v2/list?' + reqSpec; + + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); } -exports.copyOp = function(srcBucket, srcKey, destBucket, destKey, options) { - options = options || {}; - var encodedEntryURISrc = util.encodedEntry(srcBucket, srcKey); - var encodedEntryURIDest = util.encodedEntry(destBucket, destKey); - var op = "/copy/" + encodedEntryURISrc + "/" + encodedEntryURIDest; - if (options.force) { - op += "/force/true"; - } - return op; +// 批量文件管理请求，支持stat，chgm，chtype，delete，copy，move +BucketManager.prototype.batch = function (operations, callbackFunc) { + if (!operations.length) { + callbackFunc(new Error('Empty operations'), null, null) + } + + let bucket; + for (const op of operations) { + const [, , entry] = op.split('/'); + if (!entry) { + continue; + } + [bucket] = util.decodedEntry(entry); + if (bucket) { + break; + } + } + if (!bucket) { + callbackFunc(new Error('Empty bucket')); + return; + } + + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + batchReq(ctx.mac, ctx.config, operations, callbackFunc); + }); +}; + +function batchReq (mac, config, operations, callbackFunc) { + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const requestURI = scheme + config.zone.rsHost + '/batch'; + const reqParams = { + op: operations + }; + const reqBody = querystring.stringify(reqParams); + rpc.postWithOptions( + requestURI, + reqBody, + { + mac: mac + }, + callbackFunc + ); } +// 批量操作支持的指令构造器 +exports.statOp = function (bucket, key) { + return '/stat/' + util.encodedEntry(bucket, key); +}; + +exports.deleteOp = function (bucket, key) { + return '/delete/' + util.encodedEntry(bucket, key); +}; + +exports.deleteAfterDaysOp = function (bucket, key, days) { + var encodedEntryURI = util.encodedEntry(bucket, key); + return '/deleteAfterDays/' + encodedEntryURI + '/' + days; +}; + +exports.setObjectLifecycleOp = function (bucket, key, options) { + const encodedEntry = util.encodedEntry(bucket, key); + let result = '/lifecycle/' + encodedEntry + + '/toIAAfterDays/' + (options.toIaAfterDays || 0) + + '/toArchiveIRAfterDays/' + (options.toArchiveIRAfterDays || 0) + + '/toArchiveAfterDays/' + (options.toArchiveAfterDays || 0) + + '/toDeepArchiveAfterDays/' + (options.toDeepArchiveAfterDays || 0) + + '/deleteAfterDays/' + (options.deleteAfterDays || 0); + if (options.cond) { + const condStr = Object.keys(options.cond) + .reduce(function (acc, key) { + acc.push(key + '=' + options.cond[key]); + return acc; + }, []) + .join('&'); + result += '/cond/' + util.urlsafeBase64Encode(condStr); + } + return result; +}; + +exports.changeMimeOp = function (bucket, key, newMime) { + var encodedEntryURI = util.encodedEntry(bucket, key); + var encodedMime = util.urlsafeBase64Encode(newMime); + return '/chgm/' + encodedEntryURI + '/mime/' + encodedMime; +}; + +exports.changeHeadersOp = function (bucket, key, headers) { + var encodedEntryURI = util.encodedEntry(bucket, key); + var prefix = 'x-qn-meta-!'; + var path = '/chgm/' + encodedEntryURI; + for (var headerKey in headers) { + var encodedValue = util.urlsafeBase64Encode(headers[headerKey]); + var prefixedHeaderKey = prefix + headerKey; + path += '/' + prefixedHeaderKey + '/' + encodedValue; + } + + return path; +}; + +exports.changeTypeOp = function (bucket, key, newType) { + var encodedEntryURI = util.encodedEntry(bucket, key); + return '/chtype/' + encodedEntryURI + '/type/' + newType; +}; + +exports.changeStatusOp = function (bucket, key, newStatus) { + var encodedEntryURI = util.encodedEntry(bucket, key); + return '/chstatus/' + encodedEntryURI + '/status/' + newStatus; +}; + +exports.moveOp = function (srcBucket, srcKey, destBucket, destKey, options) { + options = options || {}; + var encodedEntryURISrc = util.encodedEntry(srcBucket, srcKey); + var encodedEntryURIDest = util.encodedEntry(destBucket, destKey); + var op = '/move/' + encodedEntryURISrc + '/' + encodedEntryURIDest; + if (options.force) { + op += '/force/true'; + } + return op; +}; + +exports.copyOp = function (srcBucket, srcKey, destBucket, destKey, options) { + options = options || {}; + var encodedEntryURISrc = util.encodedEntry(srcBucket, srcKey); + var encodedEntryURIDest = util.encodedEntry(destBucket, destKey); + var op = '/copy/' + encodedEntryURISrc + '/' + encodedEntryURIDest; + if (options.force) { + op += '/force/true'; + } + return op; +}; + // 空间资源下载 // 获取私有空间的下载链接 @@ -690,95 +688,772 @@ exports.copyOp = function(srcBucket, srcKey, destBucket, destKey, options) { // @param fileName 原始文件名 // @param deadline 文件有效期时间戳（单位秒） // @return 私有下载链接 -BucketManager.prototype.privateDownloadUrl = function(domain, fileName, - deadline) { - var baseUrl = this.publicDownloadUrl(domain, fileName); - if (baseUrl.indexOf('?') >= 0) { - baseUrl += '&e='; - } else { - baseUrl += '?e='; - } - baseUrl += deadline; - - var signature = util.hmacSha1(baseUrl, this.mac.secretKey); - var encodedSign = util.base64ToUrlSafe(signature); - var downloadToken = this.mac.accessKey + ':' + encodedSign; - return baseUrl + '&token=' + downloadToken; -} +BucketManager.prototype.privateDownloadUrl = function (domain, fileName, + deadline) { + var baseUrl = this.publicDownloadUrl(domain, fileName); + if (baseUrl.indexOf('?') >= 0) { + baseUrl += '&e='; + } else { + baseUrl += '?e='; + } + baseUrl += deadline; + + var signature = util.hmacSha1(baseUrl, this.mac.secretKey); + var encodedSign = util.base64ToUrlSafe(signature); + var downloadToken = this.mac.accessKey + ':' + encodedSign; + return baseUrl + '&token=' + downloadToken; +}; // 获取公开空间的下载链接 // @param domain 空间绑定的域名，比如以http或https开头 // @param fileName 原始文件名 // @return 公开下载链接 -BucketManager.prototype.publicDownloadUrl = function(domain, fileName) { - return domain + '/' + encodeUrl(fileName); +BucketManager.prototype.publicDownloadUrl = function (domain, fileName) { + return domain + '/' + encodeUrl(fileName); +}; + +// 修改文件状态 +// @link https://developer.qiniu.com/kodo/api/4173/modify-the-file-status +// @param bucket 空间名称 +// @param key 文件名称 +// @param status 文件状态 +// @param callbackFunc(err, respBody, respInfo) 回调函数 +// updateObjectStatus(bucketName string, key string, status ObjectStatus, condition UpdateObjectInfoCondition) +BucketManager.prototype.updateObjectStatus = function (bucket, key, status, + callbackFunc) { + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + updateStatusReq(ctx.mac, ctx.config, bucket, key, status, callbackFunc); + }); +}; + +function updateStatusReq(mac, config, bucket, key, status, callbackFunc) { + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var changeStatusOp = exports.changeStatusOp(bucket, key, status); + var requestURI = scheme + config.zone.rsHost + changeStatusOp; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); +} + +// 列举bucket +// @link https://developer.qiniu.com/kodo/api/3926/get-service +// @param callbackFunc(err, respBody, respInfo) 回调函数 +BucketManager.prototype.listBucket = function (callbackFunc) { + var requestURI = 'https://rs.qbox.me/buckets'; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 获取bucket信息 +// @param bucket 空间名 +// @param callbackFunc(err, respBody, respInfo) 回调函数 +BucketManager.prototype.getBucketInfo = function (bucket, callbackFunc) { + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/v2/bucketInfo?bucket=' + bucket; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +/** + * rules/add 增加 bucket 规则 + * @param { string } bucket 空间名 + * + * @param { Object } options - 配置项 + * @param { string } options.name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param { string } options.prefix - 同一个 bucket 里面前缀不能重复 + * @param { number } options.to_line_after_days - 指定文件上传多少天后转低频存储。指定为0表示不转低频存储 + * @param { number } options.to_archive_ir_after_days - 指定文件上传多少天后转归档直读存储。指定为0表示不转归档直读 + * @param { number } options.to_archive_after_days - 指定文件上传多少天后转归档存储。指定为0表示不转归档存储 + * @param { number } options.to_deep_archive_after_days - 指定文件上传多少天后转深度归档存储。指定为0表示不转深度归档存储 + * @param { number } options.delete_after_days - 指定上传文件多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param { number } options.history_delete_after_days - 指定文件成为历史版本多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param { number } options.history_to_line_after_days - 指定文件成为历史版本多少天后转低频存储。指定为0表示不转低频存储 + * + * @param { function } callbackFunc - 回调函数 + */ +BucketManager.prototype.putBucketLifecycleRule = function (bucket, options, + callbackFunc) { + PutBucketLifecycleRule(this.mac, this.config, bucket, options, callbackFunc); +}; + +function PutBucketLifecycleRule (mac, config, bucket, options, callbackFunc) { + options = options || {}; + const reqParams = { + bucket: bucket, + name: options.name + }; + + if (options.prefix) { + reqParams.prefix = options.prefix; + } else { + reqParams.prefix = ''; + } + + if (options.to_line_after_days) { + reqParams.to_line_after_days = options.to_line_after_days; + } else { + reqParams.to_line_after_days = 0; + } + + if (options.to_archive_ir_after_days) { + reqParams.to_archive_ir_after_days = options.to_archive_ir_after_days; + } else { + reqParams.to_archive_ir_after_days = 0; + } + if (options.to_archive_after_days) { + reqParams.to_archive_after_days = options.to_archive_after_days; + } else { + reqParams.to_archive_after_days = 0; + } + + if (options.to_deep_archive_after_days) { + reqParams.to_deep_archive_after_days = options.to_deep_archive_after_days; + } else { + reqParams.to_deep_archive_after_days = 0; + } + + if (options.delete_after_days) { + reqParams.delete_after_days = options.delete_after_days; + } else { + reqParams.delete_after_days = 0; + } + + if (options.history_delete_after_days) { + reqParams.history_delete_after_days = options.history_delete_after_days; + } else { + reqParams.history_delete_after_days = 0; + } + + if (options.history_to_line_after_days) { + reqParams.history_to_line_after_days = options.history_to_line_after_days; + } else { + reqParams.history_to_line_after_days = 0; + } + + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const reqSpec = querystring.stringify(reqParams); + const requestURI = scheme + conf.UC_HOST + '/rules/add?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); } -// 上传策略 -// @link https://developer.qiniu.com/kodo/manual/1206/put-policy -function PutPolicy(options) { - if (typeof options !== 'object') { - throw new Error('invalid putpolicy options'); - } - - this.scope = options.scope || null; - this.isPrefixalScope = options.isPrefixalScope || null; - this.expires = options.expires || 3600; - this.insertOnly = options.insertOnly || null; - - this.saveKey = options.saveKey || null; - this.endUser = options.endUser || null; - - this.returnUrl = options.returnUrl || null; - this.returnBody = options.returnBody || null; - - this.callbackUrl = options.callbackUrl || null; - this.callbackHost = options.callbackHost || null; - this.callbackBody = options.callbackBody || null; - this.callbackBodyType = options.callbackBodyType || null; - this.callbackFetchKey = options.callbackFetchKey || null; - - this.persistentOps = options.persistentOps || null; - this.persistentNotifyUrl = options.persistentNotifyUrl || null; - this.persistentPipeline = options.persistentPipeline || null; - - this.fsizeLimit = options.fsizeLimit || null; - this.fsizeMin = options.fsizeMin || null; - this.mimeLimit = options.mimeLimit || null; - - this.detectMime = options.detectMime || null; - this.deleteAfterDays = options.deleteAfterDays || null; - this.fileType = options.fileType || null; +/** rules/delete 删除 bucket 规则 + * @param { string } bucket - 空间名 + * @param { string } name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线 + * @param { function } callbackFunc - 回调函数 + */ +BucketManager.prototype.deleteBucketLifecycleRule = function (bucket, name, callbackFunc) { + const reqParams = { + bucket: bucket, + name: name + }; + const scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + const reqSpec = querystring.stringify(reqParams); + const requestURI = scheme + conf.UC_HOST + '/rules/delete?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +/** rules/update 更新 bucket 规则 + * @param bucket 空间名 + * + * @param { Object } options - 配置项 + * @param { string } options.name - 规则名称 bucket 内唯一，长度小于50，不能为空，只能为字母、数字、下划线: + * @param { string } options.prefix - 同一个 bucket 里面前缀不能重复 + * @param { number } options.to_line_after_days - 指定文件上传多少天后转低频存储。指定为0表示不转低频存储 + * @param { number } options.to_archive_ir_after_days - 指定文件上传多少天后转归档直读存储。指定为0表示不转归档直读存储 + * @param { number } options.to_archive_after_days - 指定文件上传多少天后转归档存储。指定为0表示不转归档存储 + * @param { number } options.to_deep_archive_after_days - 指定文件上传多少天后转深度归档存储。指定为0表示不转深度归档存储 + * @param { number } options.delete_after_days - 指定上传文件多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param { number } options.history_delete_after_days - 指定文件成为历史版本多少天后删除，指定为0表示不删除，大于0表示多少天后删除 + * @param { number } options.history_to_line_after_days - 指定文件成为历史版本多少天后转低频存储。指定为0表示不转低频存储 + * + * @param { function } callbackFunc - 回调函数 + */ +BucketManager.prototype.updateBucketLifecycleRule = function (bucket, options, callbackFunc) { + options = options || {}; + const reqParams = { + bucket: bucket, + name: options.name + }; + + if (options.prefix) { + reqParams.prefix = options.prefix; + } + + if (options.to_line_after_days) { + reqParams.to_line_after_days = options.to_line_after_days; + } + + if (options.to_archive_ir_after_days) { + reqParams.to_archive_ir_after_days = options.to_archive_ir_after_days; + } + + if (options.to_archive_after_days) { + reqParams.to_archive_after_days = options.to_archive_after_days; + } + + if (options.to_deep_archive_after_days) { + reqParams.to_deep_archive_after_days = options.to_deep_archive_after_days; + } + + if (options.delete_after_days) { + reqParams.delete_after_days = options.delete_after_days; + } + + if (options.history_delete_after_days) { + reqParams.history_delete_after_days = options.history_delete_after_days; + } + + if (options.history_to_line_after_days) { + reqParams.history_to_line_after_days = options.history_to_line_after_days; + } + + const scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + const reqSpec = querystring.stringify(reqParams); + const requestURI = scheme + conf.UC_HOST + '/rules/update?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +/** rules/get - 获取 bucket 规则 + * @param { string } bucket - 空间名 + * @param { function } callbackFunc - 回调函数 + */ +BucketManager.prototype.getBucketLifecycleRule = function (bucket, callbackFunc) { + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/rules/get?bucket=' + bucket; + rpc.getWithOptions( + requestURI, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// events/add 增加事件通知规则 +BucketManager.prototype.putBucketEvent = function (bucket, options, callbackFunc) { + PutBucketEvent(this.mac, this.config, options, bucket, callbackFunc); +}; + +function PutBucketEvent(mac, config, options, bucket, callbackFunc) { + options = options || {}; + var reqParams = { // 必填参数 + bucket: bucket, + name: options.name, + event: options.event, + callbackURL: options.callbackURL + }; + + if (options.prefix) { + reqParams.prefix = options.prefix; + } else { + reqParams.prefix = ''; + } + + if (options.suffix) { + reqParams.suffix = options.suffix; + } else { + reqParams.suffix = ''; + } + + if (options.access_key) { + reqParams.access_key = options.access_key; + } else { + reqParams.access_key = ''; + } + + if (options.host) { + reqParams.host = options.host; + } else { + reqParams.host = ''; + } + + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = querystring.stringify(reqParams); + var requestURI = scheme + conf.UC_HOST + '/events/add?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); } -PutPolicy.prototype.getFlags = function() { - var flags = {}; - var attrs = ['scope', 'isPrefixalScope', 'insertOnly', 'saveKey', 'endUser', - 'returnUrl', 'returnBody', 'callbackUrl', 'callbackHost', - 'callbackBody', 'callbackBodyType', 'callbackFetchKey', 'persistentOps', - 'persistentNotifyUrl', 'persistentPipeline', 'fsizeLimit', 'fsizeMin', - 'detectMime', 'mimeLimit', 'deleteAfterDays', 'fileType' - ]; +// events/get 更新事件通知规则 +BucketManager.prototype.updateBucketEvent = function (bucket, options, callbackFunc) { + UpdateBucketEvent(this.mac, this.config, options, bucket, callbackFunc); +}; + +function UpdateBucketEvent(mac, config, options, bucket, callbackFunc) { + options = options || {}; + var reqParams = { + bucket: bucket, + name: options.name + }; - for (var i = attrs.length - 1; i >= 0; i--) { - if (this[attrs[i]] !== null) { - flags[attrs[i]] = this[attrs[i]]; + if (options.prefix) { + reqParams.prefix = options.prefix; } - } - flags['deadline'] = this.expires + Math.floor(Date.now() / 1000); + if (options.suffix) { + reqParams.suffix = options.suffix; + } + + if (options.event) { + reqParams.event = options.event; + } + + if (options.callbackURL) { + reqParams.callbackURL = options.callbackURL; + } + + if (options.access_key) { + reqParams.access_key = options.access_key; + } + + if (options.host) { + reqParams.host = options.host; + } + + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = querystring.stringify(reqParams); + var requestURI = scheme + conf.UC_HOST + '/events/update?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); +} + +// events/get 获取事件通知规则 +BucketManager.prototype.getBucketEvent = function (bucket, callbackFunc) { + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/events/get?bucket=' + bucket; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// events/delete 删除事件通知规则 +BucketManager.prototype.deleteBucketEvent = function (bucket, name, callbackFunc) { + var reqParams = { + bucket: bucket, + name: name + }; + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = querystring.stringify(reqParams); + var requestURI = scheme + conf.UC_HOST + '/events/delete?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 设置防盗链 +// @param bucket: bucket 名 +// @param mode 0: 表示关闭Referer; 1: 表示设置Referer白名单; 2: 表示设置Referer黑名单 +// @param norefer 0: 表示不允许空 Refer 访问; 1: 表示允许空 Refer 访问 +// @param pattern 一种为空主机头域名, 比如 foo.com; 一种是泛域名, 比如 *.bar.com; +// 一种是完全通配符, 即一个 *; 多个规则之间用;隔开 +// @param source_enabled=: 源站是否支持，默认为0只给CDN配置, 设置为1表示开启源站防盗链 +BucketManager.prototype.putReferAntiLeech = function (bucket, options, callbackFunc) { + PutReferAntiLeech(this.mac, this.config, bucket, options, callbackFunc); +}; + +function PutReferAntiLeech(mac, config, bucket, options, callbackFunc) { + options = options || {}; + var reqParams = { + bucket: bucket + }; + + if (options.mode) { + reqParams.mode = options.mode; + } else { + reqParams.mode = 0; + } + + if (options.norefer) { + reqParams.norefer = options.norefer; + } else { + reqParams.norefer = 0; + } + + if (options.pattern) { + reqParams.pattern = options.pattern; + } else { + reqParams.pattern = '*'; + } + + if (options.source_enabled) { + reqParams.source_enabled = options.source_enabled; + } else { + reqParams.source_enabled = 0; + } + + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = querystring.stringify(reqParams); + var requestURI = scheme + conf.UC_HOST + '/referAntiLeech?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); +} + +/// corsRules/set 设置bucket的cors（跨域）规则 +BucketManager.prototype.putCorsRules = function (bucket, body, callbackFunc) { + PutCorsRules(this.mac, this.config, bucket, body, callbackFunc); +}; + +function PutCorsRules(mac, config, bucket, body, callbackFunc) { + var reqBody = JSON.stringify(body); + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/corsRules/set/' + bucket; + rpc.postWithOptions( + requestURI, + reqBody, + { + mac: mac + }, + callbackFunc + ); +} + +/// corsRules/get 获取bucket跨域 +BucketManager.prototype.getCorsRules = function (bucket, callbackFunc) { + GetCorsRules(this.mac, this.config, bucket, callbackFunc); +}; + +function GetCorsRules(mac, config, bucket, callbackFunc) { + var scheme = config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/corsRules/get/' + bucket; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); +} + +// BucketManager.prototype.getBucketSourceConfig = function(body, callbackFunc) { +// var reqBody = JSON.stringify(body); +// console.log(reqBody); +// var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; +// var requestURI = scheme + conf.UC_HOST + '/mirrorConfig/get'; +// var digest = util.generateAccessTokenV2(this.mac, requestURI, 'POST', conf.FormMimeJson, reqBody); +// rpc.postWithForm(requestURI, reqBody,digest, callbackFunc); +// } + +// 原图保护 +// @param bucket 空间名称 +// @param mode 为1表示开启原图保护，0表示关闭 +BucketManager.prototype.putBucketAccessStyleMode = function (bucket, mode, callbackFunc) { + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/accessMode/' + bucket + '/mode/' + mode; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 设置Bucket的cache-control: max-age属性 +// @param maxAge:为0或者负数表示为默认值（31536000） +BucketManager.prototype.putBucketMaxAge = function (bucket, options, callbackFunc) { + var maxAge = options.maxAge; + if (maxAge <= 0) { + maxAge = 31536000; + } + var reqParams = { + bucket: bucket, + maxAge: maxAge + }; + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = querystring.stringify(reqParams); + var requestURI = scheme + conf.UC_HOST + '/maxAge?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 设置Bucket私有属性 +// @param private为0表示公开，为1表示私有 +BucketManager.prototype.putBucketAccessMode = function (bucket, options, callbackFunc) { + options = options || {}; + var reqParams = { + bucket: bucket + }; + + if (options.private) { + reqParams.private = options.private; + } else { + reqParams.private = 0; + } + + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = querystring.stringify(reqParams); + var requestURI = scheme + conf.UC_HOST + '/private?' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 设置配额 +// @param bucket: 空间名称，不支持授权空间 +// @param size: 空间存储量配额,参数传入0或不传表示不更改当前配置，传入-1表示取消限额，新创建的空间默认没有限额。 +// @param count: 空间文件数配额，参数含义同 +BucketManager.prototype.putBucketQuota = function (bucket, options, callbackFunc) { + options = options || {}; + var reqParams = { + bucket: bucket + }; + + if (options.size) { + reqParams.size = options.size; + } else { + reqParams.size = 0; + } + + if (options.count) { + reqParams.count = options.count; + } else { + reqParams.count = 0; + } - return flags; + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var reqSpec = `${reqParams.bucket}/size/${reqParams.size}/count/${reqParams.count}`; + var requestURI = scheme + conf.UC_HOST + '/setbucketquota/' + reqSpec; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 获取配额 +// @param bucket: 空间名称，不支持授权空间 +BucketManager.prototype.getBucketQuota = function (bucket, callbackFunc) { + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/getbucketquota/' + bucket; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 获得Bucket的所有域名 +// @param bucket:bucketName +BucketManager.prototype.listBucketDomains = function (bucket, callbackFunc) { + var scheme = this.config.useHttpsDomain ? 'https://' : 'http://'; + var requestURI = scheme + conf.UC_HOST + '/v3/domains?tbl=' + bucket; + rpc.postWithOptions( + requestURI, + null, + { + mac: this.mac + }, + callbackFunc + ); +}; + +// 解冻归档存储文件 +BucketManager.prototype.restoreAr = function (entry, freezeAfterDays, callbackFunc) { + const [bucket] = entry.split(':'); + util.prepareZone(this, this.mac.accessKey, bucket, function (err, ctx) { + if (err) { + callbackFunc(err, null, null); + return; + } + restoreArReq(ctx.mac, ctx.config, entry, freezeAfterDays, callbackFunc); + }); +}; + +function restoreArReq (mac, config, entry, freezeAfterDays, callbackFunc) { + const scheme = config.useHttpsDomain ? 'https://' : 'http://'; + const requestURI = scheme + config.zone.rsHost + '/restoreAr/' + util.urlsafeBase64Encode(entry) + '/freezeAfterDays/' + freezeAfterDays; + rpc.postWithOptions( + requestURI, + null, + { + mac: mac + }, + callbackFunc + ); } +// just for compatibility with old sdk versions +function _putPolicyBuildInKeys () { + return ['scope', 'isPrefixalScope', 'insertOnly', 'saveKey', 'forceSaveKey', + 'endUser', 'returnUrl', 'returnBody', 'callbackUrl', 'callbackHost', + 'callbackBody', 'callbackBodyType', 'callbackFetchKey', 'persistentOps', + 'persistentNotifyUrl', 'persistentPipeline', 'fsizeLimit', 'fsizeMin', + 'detectMime', 'mimeLimit', 'deleteAfterDays', 'fileType' + ]; +} -PutPolicy.prototype.uploadToken = function(mac) { - mac = mac || new digest.Mac(); - var flags = this.getFlags(); - var encodedFlags = util.urlsafeBase64Encode(JSON.stringify(flags)); - var encoded = util.hmacSha1(encodedFlags, mac.secretKey); - var encodedSign = util.base64ToUrlSafe(encoded); - var uploadToken = mac.accessKey + ':' + encodedSign + ':' + encodedFlags; - return uploadToken; +/** + * @typedef PutPolicyOptions + * @extends Object. + * @property {string} scope + * @property {number} [isPrefixalScope] + * @property {number} [expires] + * @property {number} [insertOnly] + * @property {string} [saveKey] + * @property {string} [forceSaveKey] + * @property {string} [endUser] + * @property {string} [returnUrl] + * @property {string} [returnBody] + * @property {string} [callbackUrl] + * @property {string} [callbackHost] + * @property {string} [callbackBody] + * @property {string} [callbackBodyType] + * @property {number} [callbackFetchKey] + * @property {string} [persistentOps] + * @property {string} [persistentNotifyUrl] + * @property {string} [persistentPipeline] + * @property {number} [fsizeLimit] + * @property {number} [fsizeMin] + * @property {string} [mimeLimit] + * @property {number} [detectMime] + * @property {number} [deleteAfterDays] + * @property {number} [fileType] + * @property {string} [transform] Deprecated + * @property {string} [transformFallbackMode] Deprecated + * @property {string} [transformFallbackKey] Deprecated + */ + +/** + * 上传策略 + * @link https://developer.qiniu.com/kodo/manual/1206/put-policy + * @param {PutPolicyOptions} options + * @constructor + * @extends Object. + */ +function PutPolicy (options) { + if (typeof options !== 'object') { + throw new Error('invalid putpolicy options'); + } + + Object.keys(options).forEach(k => { + if (k === 'expires') { + return; + } + this[k] = options[k]; + }); + + this.expires = options.expires || 3600; + _putPolicyBuildInKeys().forEach(k => { + if (this[k] === undefined) { + this[k] = this[k] || null; + } + }); } + +PutPolicy.prototype.getFlags = function () { + const flags = {}; + + Object.keys(this).forEach(k => { + if (k === 'expires' || this[k] === null) { + return; + } + flags[k] = this[k]; + }); + + flags.deadline = this.expires + Math.floor(Date.now() / 1000); + + return flags; +}; + +PutPolicy.prototype.uploadToken = function (mac) { + mac = mac || new digest.Mac(); + const flags = this.getFlags(); + const encodedFlags = util.urlsafeBase64Encode(JSON.stringify(flags)); + const encoded = util.hmacSha1(encodedFlags, mac.secretKey); + const encodedSign = util.base64ToUrlSafe(encoded); + return [ + mac.accessKey, + encodedSign, + encodedFlags + ].join(':'); +}; diff --git a/qiniu/util.js b/qiniu/util.js index 126dead5..3c492f38 100644 --- a/qiniu/util.js +++ b/qiniu/util.js @@ -1,130 +1,307 @@ -var url = require('url'); -var crypto = require('crypto'); -var conf = require('./conf'); +const url = require('url'); +const crypto = require('crypto'); +const zone = require('./zone'); // Check Timestamp Expired or not -exports.isTimestampExpired = function(timestamp) { - return timestamp < parseInt(Date.now() / 1000); -} +exports.isTimestampExpired = function (timestamp) { + return timestamp < Math.trunc(Date.now() / 1000); +}; + +// Format Data +exports.formatDateUTC = function (date, layout) { + function pad (num, digit) { + const d = digit || 2; + let result = num.toString(); + while (result.length < d) { + result = '0' + result; + } + return result; + } + + const d = new Date(date); + const year = d.getUTCFullYear(); + const month = d.getUTCMonth() + 1; + const day = d.getUTCDate(); + const hour = d.getUTCHours(); + const minute = d.getUTCMinutes(); + const second = d.getUTCSeconds(); + const millisecond = d.getUTCMilliseconds(); + + let result = layout || 'YYYY-MM-DDTHH:MM:ss.SSSZ'; + + result = result.replace(/YYYY/g, year.toString()) + .replace(/MM/g, pad(month)) + .replace(/DD/g, pad(day)) + .replace(/HH/g, pad(hour)) + .replace(/mm/g, pad(minute)) + .replace(/ss/g, pad(second)) + .replace(/SSS/g, pad(millisecond, 3)); + + return result; +}; // Encoded Entry -exports.encodedEntry = function(bucket, key) { - return exports.urlsafeBase64Encode(bucket + (key ? ':' + key : '')); -} +exports.encodedEntry = function (bucket, key) { + let strToEncode = bucket; + if (key !== undefined) { + strToEncode += ':' + key; + } + return exports.urlsafeBase64Encode(strToEncode); +}; + +exports.decodedEntry = function (entry) { + const [bucket, key] = exports.urlSafeBase64Decode(entry).split(':'); + return [bucket, key]; +}; // Get accessKey from uptoken -exports.getAKFromUptoken = function(uploadToken) { - var sepIndex = uploadToken.indexOf(":"); - return uploadToken.substring(0, sepIndex); -} +exports.getAKFromUptoken = function (uploadToken) { + var sepIndex = uploadToken.indexOf(':'); + return uploadToken.substring(0, sepIndex); +}; // Get bucket from uptoken -exports.getBucketFromUptoken = function(uploadToken) { - var sepIndex = uploadToken.lastIndexOf(":"); - var encodedPutPolicy = uploadToken.substring(sepIndex + 1); - var putPolicy = exports.urlSafeBase64Decode(encodedPutPolicy); - var putPolicyObj = JSON.parse(putPolicy); - var scope = putPolicyObj.scope; - var scopeSepIndex = scope.indexOf(":"); - if (scopeSepIndex == -1) { - return scope; - } else { - return scope.substring(0, scopeSepIndex); - } -} +exports.getBucketFromUptoken = function (uploadToken) { + var sepIndex = uploadToken.lastIndexOf(':'); + var encodedPutPolicy = uploadToken.substring(sepIndex + 1); + var putPolicy = exports.urlSafeBase64Decode(encodedPutPolicy); + var putPolicyObj = JSON.parse(putPolicy); + var scope = putPolicyObj.scope; + var scopeSepIndex = scope.indexOf(':'); + if (scopeSepIndex == -1) { + return scope; + } else { + return scope.substring(0, scopeSepIndex); + } +}; +exports.base64ToUrlSafe = function (v) { + return v.replace(/\//g, '_').replace(/\+/g, '-'); +}; -exports.base64ToUrlSafe = function(v) { - return v.replace(/\//g, '_').replace(/\+/g, '-'); -} - -exports.urlSafeToBase64 = function(v) { - return v.replace(/\_/g, '/').replace(/\-/g, '+'); -} +exports.urlSafeToBase64 = function (v) { + return v.replace(/_/g, '/').replace(/-/g, '+'); +}; // UrlSafe Base64 Decode -exports.urlsafeBase64Encode = function(jsonFlags) { - var encoded = new Buffer(jsonFlags).toString('base64'); - return exports.base64ToUrlSafe(encoded); -} +exports.urlsafeBase64Encode = function (jsonFlags) { + var encoded = Buffer.from(jsonFlags).toString('base64'); + return exports.base64ToUrlSafe(encoded); +}; // UrlSafe Base64 Decode -exports.urlSafeBase64Decode = function(fromStr) { - return new Buffer(exports.urlSafeToBase64(fromStr), 'base64').toString(); -} +exports.urlSafeBase64Decode = function (fromStr) { + return Buffer.from(exports.urlSafeToBase64(fromStr), 'base64').toString(); +}; // Hmac-sha1 Crypt -exports.hmacSha1 = function(encodedFlags, secretKey) { - /* +exports.hmacSha1 = function (encodedFlags, secretKey) { + /* *return value already encoded with base64 * */ - var hmac = crypto.createHmac('sha1', secretKey); - hmac.update(encodedFlags); - return hmac.digest('base64'); -} + var hmac = crypto.createHmac('sha1', secretKey); + hmac.update(encodedFlags); + return hmac.digest('base64'); +}; + +// get md5 +exports.getMd5 = function (data) { + var md5 = crypto.createHash('md5'); + return md5.update(data).digest('hex'); +}; // 创建 AccessToken 凭证 // @param mac AK&SK对象 // @param requestURI 请求URL // @param reqBody 请求Body，仅当请求的 ContentType 为 // application/x-www-form-urlencoded时才需要传入该参数 -exports.generateAccessToken = function(mac, requestURI, reqBody) { - var u = url.parse(requestURI); - var path = u.path; - var access = path + '\n'; - - if (reqBody) { - access += reqBody; - } - - var digest = exports.hmacSha1(access, mac.secretKey); - var safeDigest = exports.base64ToUrlSafe(digest); - return 'QBox ' + mac.accessKey + ':' + safeDigest; +exports.generateAccessToken = function (mac, requestURI, reqBody) { + var u = new url.URL(requestURI); + var path = u.pathname + u.search; + var access = path + '\n'; + + if (reqBody) { + access += reqBody; + } + + var digest = exports.hmacSha1(access, mac.secretKey); + var safeDigest = exports.base64ToUrlSafe(digest); + return 'QBox ' + mac.accessKey + ':' + safeDigest; +}; + +const isTokenTable = { + '!': true, + '#': true, + $: true, + '%': true, + '&': true, + '\\': true, + '*': true, + '+': true, + '-': true, + '.': true, + 0: true, + 1: true, + 2: true, + 3: true, + 4: true, + 5: true, + 6: true, + 7: true, + 8: true, + 9: true, + A: true, + B: true, + C: true, + D: true, + E: true, + F: true, + G: true, + H: true, + I: true, + J: true, + K: true, + L: true, + M: true, + N: true, + O: true, + P: true, + Q: true, + R: true, + S: true, + T: true, + U: true, + W: true, + V: true, + X: true, + Y: true, + Z: true, + '^': true, + _: true, + '`': true, + a: true, + b: true, + c: true, + d: true, + e: true, + f: true, + g: true, + h: true, + i: true, + j: true, + k: true, + l: true, + m: true, + n: true, + o: true, + p: true, + q: true, + r: true, + s: true, + t: true, + u: true, + v: true, + w: true, + x: true, + y: true, + z: true, + '|': true, + '~': true +}; +/** + * 是否合法的 header field name 字符 + * @param ch string + * @return boolean|undefined + */ +function validHeaderKeyChar (ch) { + if (ch.charCodeAt(0) >= 128) { + return false; + } + return isTokenTable[ch]; } +/** + * 规范化 header field name + * @param fieldName string + * @return string + */ +exports.canonicalMimeHeaderKey = function (fieldName) { + for (const ch of fieldName) { + if (!validHeaderKeyChar(ch)) { + return fieldName; + } + } + return fieldName.split('-') + .map(function (text) { + return text.substring(0, 1).toUpperCase() + text.substring(1).toLowerCase(); + }) + .join('-'); +}; + // 创建 AccessToken 凭证 // @param mac AK&SK对象 // @param requestURI 请求URL // @param reqMethod 请求方法，例如 GET，POST // @param reqContentType 请求类型，例如 application/json 或者 application/x-www-form-urlencoded -// @param reqBody 请求Body，仅当请求的 ContentType 为 application/json 或者 +// @param reqBody 请求Body，仅当请求的 ContentType 为 application/json 或者 // application/x-www-form-urlencoded 时才需要传入该参数 -exports.generateAccessTokenV2 = function (mac, requestURI, reqMethod, reqContentType, reqBody) { - var u = url.parse(requestURI); - var path = u.path; - var query = u.query; - var host = u.host; - var port = u.port; - - var access = reqMethod.toUpperCase() + ' ' + path; - if (query) { - access += '?' + query; - } - // add host - access += '\nHost: ' + host; - // add port - if (port) { - access += ':' + port; - } - - // add content type - if (reqContentType && (reqContentType=="application/json" || reqContentType=="application/x-www-form-urlencoded")) { - access += '\nContent-Type: ' + reqContentType; - } - - access += '\n\n'; - - // add reqbody - if (reqBody) { - access += reqBody; - } - - console.log(access); - - var digest = exports.hmacSha1(access, mac.secretKey); - var safeDigest = exports.base64ToUrlSafe(digest); - return 'Qiniu ' + mac.accessKey + ':' + safeDigest; -} +exports.generateAccessTokenV2 = function (mac, requestURI, reqMethod, reqContentType, reqBody, reqHeaders) { + var u = new url.URL(requestURI); + var path = u.pathname; + var search = u.search; + var host = u.host; + var port = u.port; + + var access = reqMethod.toUpperCase() + ' ' + path; + if (search) { + access += search; + } + // add host + access += '\nHost: ' + host; + // add port + if (port) { + access += ':' + port; + } + + // add content type + if (reqContentType) { + access += '\nContent-Type: ' + reqContentType; + } else { + access += '\nContent-Type: application/x-www-form-urlencoded'; + } + + // add headers + if (reqHeaders) { + const canonicalHeaders = Object.keys(reqHeaders) + .reduce(function (acc, k) { + acc[exports.canonicalMimeHeaderKey(k)] = reqHeaders[k]; + return acc; + }, {}); + const headerText = Object.keys(canonicalHeaders) + .filter(function (k) { + return k.startsWith('X-Qiniu-') && k.length > 'X-Qiniu-'.length; + }) + .sort() + .map(function (k) { + return k + ': ' + canonicalHeaders[k]; + }) + .join('\n'); + if (headerText) { + access += '\n' + headerText; + } + } + + access += '\n\n'; + + // add reqbody + if (reqBody && reqContentType !== 'application/octet-stream') { + access += reqBody; + } + + var digest = exports.hmacSha1(access, mac.secretKey); + var safeDigest = exports.base64ToUrlSafe(digest); + return 'Qiniu ' + mac.accessKey + ':' + safeDigest; +}; // 校验七牛上传回调的Authorization // @param mac AK&SK对象 @@ -132,7 +309,36 @@ exports.generateAccessTokenV2 = function (mac, requestURI, reqMethod, reqContent // @param reqBody 请求Body，仅当请求的ContentType为 // application/x-www-form-urlencoded时才需要传入该参数 // @param callbackAuth 回调时请求的Authorization头部值 -exports.isQiniuCallback = function(mac, requestURI, reqBody, callbackAuth) { - var auth = exports.generateAccessToken(mac, requestURI, reqBody); - return auth === callbackAuth; -} +exports.isQiniuCallback = function (mac, requestURI, reqBody, callbackAuth) { + var auth = exports.generateAccessToken(mac, requestURI, reqBody); + return auth === callbackAuth; +}; + +exports.prepareZone = function (ctx, accessKey, bucket, callback) { + var useCache = false; + if (ctx.config.zone !== '' && ctx.config.zone != null) { + if (ctx.config.zoneExpire === -1) { + useCache = true; + } else { + if (!exports.isTimestampExpired(ctx.config.zoneExpire)) { + useCache = true; + } + } + } + + if (useCache) { + callback(null, ctx); + } else { + zone.getZoneInfo(accessKey, bucket, function (err, cZoneInfo, + cZoneExpire) { + if (err) { + callback(err); + return; + } + // update object + ctx.config.zone = cZoneInfo; + ctx.config.zoneExpire = cZoneExpire + Math.trunc(Date.now() / 1000); + callback(null, ctx); + }); + } +}; diff --git a/qiniu/zone.js b/qiniu/zone.js index 6e74c556..10ebacd2 100644 --- a/qiniu/zone.js +++ b/qiniu/zone.js @@ -1,115 +1,135 @@ -const urllib = require('urllib') -const util = require('util') const conf = require('./conf'); +const { RetryDomainsMiddleware } = require('./httpc/middleware'); +const rpc = require('./rpc'); -//huadong +// huadong exports.Zone_z0 = new conf.Zone([ - 'up.qiniup.com', - 'up-nb.qiniup.com', - 'up-xs.qiniup.com', - ], [ - 'upload.qiniup.com', - 'upload-nb.qiniup.com', - 'upload-xs.qiniup.com', - ], 'iovip.qbox.me', - 'rs.qiniu.com', - 'rsf.qiniu.com', - 'api.qiniu.com'); + 'up.qiniup.com' +], [ + 'upload.qiniup.com' +], 'iovip.qbox.me', +'rs.qbox.me', +'rsf.qbox.me', +'api.qiniuapi.com'); + +// huadong2 +exports.Zone_cn_east_2 = new conf.Zone([ + 'up-cn-east-2.qiniup.com' +], [ + 'upload-cn-east-2.qiniup.com' +], 'iovip-cn-east-2.qiniuio.com', +'rs-cn-east-2.qiniuapi.com', +'rsf-cn-east-2.qiniuapi.com', +'api-cn-east-2.qiniuapi.com'); -//huabei +// huabei exports.Zone_z1 = new conf.Zone([ - 'up-z1.qiniup.com', - ], [ - 'upload-z1.qiniup.com', - ], 'iovip-z1.qbox.me', - 'rs-z1.qiniu.com', - 'rsf-z1.qiniu.com', - 'api-z1.qiniu.com'); + 'up-z1.qiniup.com' +], [ + 'upload-z1.qiniup.com' +], 'iovip-z1.qbox.me', +'rs-z1.qbox.me', +'rsf-z1.qbox.me', +'api-z1.qiniuapi.com'); -//huanan +// huanan exports.Zone_z2 = new conf.Zone([ - 'up-z2.qiniup.com', - 'up-gz.qiniup.com', - 'up-fs.qiniup.com' - ], [ - 'upload-z2.qiniup.com', - 'upload-gz.qiniup.com', - 'upload-fs.qiniup.com', - ], 'iovip-z2.qbox.me', - 'rs-z2.qiniu.com', - 'rsf-z2.qiniu.com', - 'api-z2.qiniu.com'); - + 'up-z2.qiniup.com' +], [ + 'upload-z2.qiniup.com' +], 'iovip-z2.qbox.me', +'rs-z2.qbox.me', +'rsf-z2.qbox.me', +'api-z2.qiniuapi.com'); -//beimei +// beimei exports.Zone_na0 = new conf.Zone([ - 'up-na0.qiniup.com', - ], [ - 'upload-na0.qiniup.com', - ], 'iovip-na0.qbox.me', - 'rs-na0.qiniu.com', - 'rsf-na0.qiniu.com', - 'api-na0.qiniu.com') - + 'up-na0.qiniup.com' +], [ + 'upload-na0.qiniup.com' +], 'iovip-na0.qbox.me', +'rs-na0.qbox.me', +'rsf-na0.qbox.me', +'api-na0.qiniuapi.com'); +// singapore exports.Zone_as0 = new conf.Zone([ - 'up-as0.qiniup.com', + 'up-as0.qiniup.com' ], [ - 'upload-as0.qiniup.com', + 'upload-as0.qiniup.com' ], 'iovip-as0.qbox.me', -'rs-as0.qiniu.com', -'rsf-as0.qiniu.com', -'api-as0.qiniu.com') +'rs-as0.qbox.me', +'rsf-as0.qbox.me', +'api-as0.qiniuapi.com'); +exports.getZoneInfo = function (accessKey, bucket, callbackFunc) { + const apiAddr = 'https://' + conf.QUERY_REGION_HOST + '/v4/query'; -exports.getZoneInfo = function(accessKey, bucket, callbackFunc) { - var apiAddr = util.format('https://uc.qbox.me/v2/query?ak=%s&bucket=%s', - accessKey, bucket); - urllib.request(apiAddr, function(respErr, respData, respInfo) { - if (respErr) { - callbackFunc(respErr, null, null); - return; - } + rpc.qnHttpClient.get({ + url: apiAddr, + params: { + ak: accessKey, + bucket: bucket + }, + middlewares: [ + new RetryDomainsMiddleware({ + backupDomains: conf.QUERY_REGION_BACKUP_HOSTS + }) + ], + callback: function (respErr, respData, respInfo) { + if (respErr) { + callbackFunc(respErr, null, null); + return; + } - if (respInfo.statusCode != 200) { - //not ok - respErr = new Error(respInfo.statusCode + "\n" + respData); - callbackFunc(respErr, null, null); - return; - } + if (respInfo.statusCode !== 200) { + // not ok + respErr = new Error(respInfo.statusCode + '\n' + respData); + callbackFunc(respErr, null, null); + return; + } - var zoneData = JSON.parse(respData); - var srcUpHosts = []; - var cdnUpHosts = []; - var zoneExpire = 0; + let zoneData; + try { + const hosts = JSON.parse(respData).hosts; + if (!hosts || !hosts.length) { + respErr = new Error('no host available: ' + respData); + callbackFunc(respErr, null, null); + return; + } + zoneData = hosts[0]; + } catch (err) { + callbackFunc(err, null, null); + return; + } + let srcUpHosts = []; + let cdnUpHosts = []; + let zoneExpire = 0; - try { - zoneExpire = zoneData.ttl; - //read src hosts - zoneData.up.src.main.forEach(function(host) { - srcUpHosts.push(host); - }); - if (zoneData.up.src.backup) { - zoneData.up.src.backup.forEach(function(host) { - srcUpHosts.push(host); - }); - } + try { + zoneExpire = zoneData.ttl; + // read src hosts + srcUpHosts = zoneData.up.domains; - //read acc hosts - zoneData.up.acc.main.forEach(function(host) { - cdnUpHosts.push(host); - }); - if (zoneData.up.acc.backup) { - zoneData.up.acc.backup.forEach(function(host) { - cdnUpHosts.push(host); - }); - } + // read acc hosts + cdnUpHosts = zoneData.up.domains; - var ioHost = zoneData.io.src.main[0]; - var zoneInfo = new conf.Zone(srcUpHosts, cdnUpHosts, ioHost) - callbackFunc(null, zoneInfo, zoneExpire); - } catch (e) { - callbackFunc(e, null, null); - } - }); -} + const ioHost = zoneData.io.domains[0]; + const rsHost = zoneData.rs.domains[0]; + const rsfHost = zoneData.rsf.domains[0]; + const apiHost = zoneData.api.domains[0]; + const zoneInfo = new conf.Zone( + srcUpHosts, + cdnUpHosts, + ioHost, + rsHost, + rsfHost, + apiHost + ); + callbackFunc(null, zoneInfo, zoneExpire); + } catch (e) { + callbackFunc(e, null, null); + } + } + }); +}; diff --git a/test/cdn.test.js b/test/cdn.test.js new file mode 100644 index 00000000..a4139da5 --- /dev/null +++ b/test/cdn.test.js @@ -0,0 +1,115 @@ +const qiniu = require('../index.js'); +const should = require('should'); +const proc = require('process'); +const console = require('console'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + done(); +}); + +// eslint-disable-next-line no-undef +describe('test start cdn', function () { + this.timeout(0); + + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + var domain = proc.env.QINIU_TEST_DOMAIN; + var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + var cdnManager = new qiniu.cdn.CdnManager(mac); + + it('test getCdnLogList', function (done) { + var day = (new Date()).toISOString().substring(0, 10); + cdnManager.getCdnLogList([domain], day, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'data'); + done(); + }); + }); + + it('test getFluxData', function (done) { + var today = new Date(); + var endDay = today.toISOString().substring(0, 10); + today.setDate(today.getDate() - 30); + var startDay = today.toISOString().substring(0, 10); + cdnManager.getFluxData(startDay, endDay, '5hour', [domain], + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'data'); + done(); + }); + }); + + it('test getBandwidthData', function (done) { + var today = new Date(); + var endDay = today.toISOString().substring(0, 10); + today.setDate(today.getDate() - 30); + var startDay = today.toISOString().substring(0, 10); + cdnManager.getBandwidthData(startDay, endDay, '5hour', [domain], + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'data'); + done(); + }); + }); + + it('test prefetchUrls', function (done) { + var urls = ['http://' + domain + '/qiniu.mp4']; + cdnManager.prefetchUrls(urls, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'taskIds', 'requestId'); + done(); + }); + }); + + it('test refreshUrls', function (done) { + var urls = ['http://' + domain + '/qiniu.mp4']; + cdnManager.refreshUrls(urls, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'taskIds', 'requestId'); + done(); + }); + }); + + it('test refreshDirs', function (done) { + var dirs = ['http://' + domain + '/']; + cdnManager.refreshDirs(dirs, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'taskIds', 'requestId'); + done(); + }); + }); + + it('test refreshUrlsAndDirs', function (done) { + var urls = ['http://' + domain + '/qiniu.mp4']; + var dirs = ['http://' + domain + '/']; + cdnManager.refreshUrlsAndDirs(urls, dirs, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('code', 'taskIds', 'requestId'); + done(); + }); + }); + + it('test createTimestampAntiLeechUrl', function (done) { + var host = 'http://' + domain; + var url = cdnManager.createTimestampAntiLeechUrl(host, 'qiniu.mp4', 'aa=23', 'encryptKey', 20); + should.equal(url, host + '/qiniu.mp4?aa=23&sign=1a530a8baafe126145f49cb738a50c09&t=14'); + done(); + }); +}); diff --git a/test/fop.test.js b/test/fop.test.js index 4a9b3109..4a732eb4 100644 --- a/test/fop.test.js +++ b/test/fop.test.js @@ -1,54 +1,74 @@ - const qiniu = require("../index.js"); - const should = require('should'); - const proc = require("process"); - - before(function(done) { - if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || ! - process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { - console.log('should run command `source test-env.sh` first\n'); - process.exit(0); +const qiniu = require('../index.js'); +const should = require('should'); +const proc = require('process'); +const console = require('console'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); } done(); - }); +}); + +// eslint-disable-next-line no-undef +describe('test start fop', function () { + this.timeout(0); - describe('test start fop', function() { var accessKey = proc.env.QINIU_ACCESS_KEY; var secretKey = proc.env.QINIU_SECRET_KEY; var srcBucket = proc.env.QINIU_TEST_BUCKET; var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); var config = new qiniu.conf.Config(); - //config.useHttpsDomain = true; + config.useHttpsDomain = true; config.zone = qiniu.zone.Zone_z0; - it('test video fop', function(done) { - console.log(srcBucket); - - var pipeline = 'sdktest'; - var srcKey = 'qiniu.mp4'; - var operManager = new qiniu.fop.OperationManager(mac, config); - - //处理指令集合 - var saveBucket = srcBucket; - var fops = [ - 'avthumb/mp4/s/480x320/vb/150k|saveas/' + qiniu.util.urlsafeBase64Encode( - saveBucket + ":qiniu_480x320.mp4"), - 'vframe/jpg/offset/10|saveas/' + qiniu.util.urlsafeBase64Encode( - saveBucket + - ":qiniu_frame1.jpg") - ]; - - var options = { - 'notifyURL': 'http://api.example.com/pfop/callback', - 'force': false, - }; - - //持久化数据处理返回的是任务的persistentId，可以根据这个id查询处理状态 - operManager.pfop(srcBucket, srcKey, fops, pipeline, options, - function(err, respBody, respInfo) { - console.log(respBody); - should.not.exist(err); - respBody.should.have.keys('persistentId'); - done(); - }); + var persistentId; + + it('test video fop', function (done) { + console.log(srcBucket); + + var pipeline = 'sdktest'; + var srcKey = 'qiniu.mp4'; + var operManager = new qiniu.fop.OperationManager(mac, config); + + // 处理指令集合 + var saveBucket = srcBucket; + var fops = [ + 'avthumb/mp4/s/480x320/vb/150k|saveas/' + qiniu.util.urlsafeBase64Encode( + saveBucket + ':qiniu_480x320.mp4'), + 'vframe/jpg/offset/10|saveas/' + qiniu.util.urlsafeBase64Encode( + saveBucket + + ':qiniu_frame1.jpg') + ]; + + var options = { + notifyURL: 'http://api.example.com/pfop/callback', + force: false + }; + + // 持久化数据处理返回的是任务的persistentId，可以根据这个id查询处理状态 + operManager.pfop(srcBucket, srcKey, fops, pipeline, options, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('persistentId'); + persistentId = respBody.persistentId; + done(); + }); + }); + + it('test video prefop', function (done) { + var operManager = new qiniu.fop.OperationManager(mac, config); + // 查询处理状态 + operManager.prefop(persistentId, + function (err, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('id', 'pipeline', 'inputBucket', 'inputKey'); + respBody.should.have.property('id', persistentId); + done(); + }); }); - }); +}); diff --git a/test/form_up.test.js b/test/form_up.test.js index 2d56e1f1..3b6b5b44 100644 --- a/test/form_up.test.js +++ b/test/form_up.test.js @@ -1,156 +1,322 @@ +const os = require('os'); const path = require('path'); const should = require('should'); -const assert = require('assert'); -const qiniu = require("../index.js"); -const proc = require("process"); -const fs = require("fs"); - -before(function(done) { - if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || ! - process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { - console.log('should run command `source test-env.sh` first\n'); - process.exit(0); - } - done(); +// const assert = require('assert'); +const qiniu = require('../index.js'); +const proc = require('process'); +const fs = require('fs'); +const console = require('console'); + +// file to upload +var testFilePath_1 = path.join(os.tmpdir(), 'nodejs-sdk-test-1.bin'); +var testFilePath_2 = path.join(os.tmpdir(), 'nodejs-sdk-test-2.bin'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + var callbacked = 0; + var callback = function () { + callbacked += 1; + if (callbacked >= 2) { + done(); + } + }; + fs.createReadStream('/dev/urandom', { end: (1 << 20) * 10 }) + .pipe(fs.createWriteStream(testFilePath_1)) + .on('finish', callback); + fs.createReadStream('/dev/urandom', { end: (1 << 20) * 5 }) + .pipe(fs.createWriteStream(testFilePath_2)) + .on('finish', callback); }); +// eslint-disable-next-line no-undef +describe('test form up', function () { + this.timeout(0); -//file to upload -var testFilePath1 = path.join(__dirname, 'logo.png'); -var testFilePath2 = path.join(__dirname, 'github.png'); + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + var bucket = proc.env.QINIU_TEST_BUCKET; + var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + var config = new qiniu.conf.Config(); + config.useCdnDomain = true; + config.useHttpsDomain = true; + var bucketManager = new qiniu.rs.BucketManager(mac, config); -describe('test form up', function() { - var accessKey = proc.env.QINIU_ACCESS_KEY; - var secretKey = proc.env.QINIU_SECRET_KEY; - var bucket = proc.env.QINIU_TEST_BUCKET; - var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); - var config = new qiniu.conf.Config(); - //config.useHttpsDomain = true; - config.zone = qiniu.zone.Zone_z0; - var bucketManager = new qiniu.rs.BucketManager(mac, config); + // delete all the files uploaded + var keysToDelete = []; - //delete all the files uploaded - var keysToDelete = []; + // eslint-disable-next-line no-undef + after(function (done) { + const deleteOps = []; + keysToDelete.forEach(function (key) { + deleteOps.push(qiniu.rs.deleteOp(bucket, key)); + }); - after(function(done) { - var deleteOps = []; - keysToDelete.forEach(function(key) { - deleteOps.push(qiniu.rs.deleteOp(bucket, key)); + bucketManager.batch(deleteOps, function (respErr, respBody) { + respBody.forEach(function (ret, i) { + ret.code.should.be.eql( + 200, + JSON.stringify({ + deleteOps: deleteOps[i], + key: keysToDelete[i], + ret: ret + }) + ); + }); + done(); + }); }); - bucketManager.batch(deleteOps, function(respErr, respBody, respInfo) { - //console.log(respBody); - respBody.forEach(function(ret) { - ret.should.eql({ - code: 200 + var options = { + scope: bucket + }; + var putPolicy = new qiniu.rs.PutPolicy(options); + var uploadToken = putPolicy.uploadToken(mac); + var formUploader = new qiniu.form_up.FormUploader(config); + var putExtra = new qiniu.form_up.PutExtra(); + + // eslint-disable-next-line no-undef + describe('test form up#putStreamWithoutKey', function () { + // eslint-disable-next-line no-undef + it('test form up#putStreamWithoutKey', function () { + var key = null; + var rs = fs.createReadStream(testFilePath_1); + formUploader.putStream(uploadToken, key, rs, putExtra, + function (respErr, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); }); - }); - done(); }); - }); - - var options = { - scope: bucket, - } - var putPolicy = new qiniu.rs.PutPolicy(options); - var uploadToken = putPolicy.uploadToken(mac); - var config = new qiniu.conf.Config(); - config.zone = qiniu.zone.Zone_z0; - var formUploader = new qiniu.form_up.FormUploader(config); - var putExtra = new qiniu.form_up.PutExtra(); - - describe('test form up#putStreamWithoutKey', function() { - it('test form up#putStreamWithoutKey', function(done) { - var key = null; - var rs = fs.createReadStream(testFilePath1); - formUploader.putStream(uploadToken, key, rs, putExtra, - function(respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + // eslint-disable-next-line no-undef + describe('test form up#putStream', function () { + // eslint-disable-next-line no-undef + it('test form up#putStream', function () { + var key = 'storage_putStream_test' + Math.random(1000); + var rs = fs.createReadStream(testFilePath_1); + return formUploader.putStream(uploadToken, key, rs, putExtra, + function (respErr, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); }); }); - }); - - describe('test form up#putStream', function() { - it('test form up#putStream', function(done) { - var key = 'storage_putStream_test' + Math.random(1000); - var rs = fs.createReadStream(testFilePath1); - formUploader.putStream(uploadToken, key, rs, putExtra, - function(respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + // eslint-disable-next-line no-undef + describe('test form up#put', function () { + // eslint-disable-next-line no-undef + it('test form up#put', function () { + var key = 'storage_put_test' + Math.random(1000); + return formUploader.put(uploadToken, key, 'hello world', putExtra, + function (respErr, + respBody) { + // console.log(respBody); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); }); }); - }); - - describe('test form up#put', function() { - it('test form up#put', function(done) { - var key = 'storage_put_test' + Math.random(1000); - formUploader.put(uploadToken, key, "hello world", putExtra, - function(respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + // eslint-disable-next-line no-undef + describe('test form up#putWithoutKey', function () { + // eslint-disable-next-line no-undef + it('test form up#putWithoutKey', function () { + return formUploader.putWithoutKey(uploadToken, 'hello world', + putExtra, + function (respErr, + respBody) { + // console.log(respBody); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); }); }); - }); - - describe('test form up#putWithoutKey', function() { - it('test form up#putWithoutKey', function(done) { - formUploader.putWithoutKey(uploadToken, "hello world", - putExtra, - function(respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + // eslint-disable-next-line no-undef + describe('test form up#putFile', function () { + // eslint-disable-next-line no-undef + it('test form up#putFile', function () { + var key = 'storage_putFile_test' + Math.random(1000); + return formUploader.putFile(uploadToken, key, testFilePath_2, + putExtra, + function ( + respErr, + respBody) { + // console.log(respBody); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); }); }); - }); - - describe('test form up#putFile', function() { - it('test form up#putFile', function(done) { - var key = 'storage_putFile_test' + Math.random(1000); - formUploader.putFile(uploadToken, key, testFilePath2, - putExtra, - function( - respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + describe('test form up#putFile with double quotes', function () { + it('test form up#putFile with double quotes', function () { + const key = 'storage_putFile_"test"' + Math.ceil(1000 * Math.random()); + const putExtra = new qiniu.form_up.PutExtra(); + putExtra.fname = key; + return formUploader.putFile(uploadToken, key, testFilePath_2, + putExtra, + function ( + respErr, + respBody) { + // console.log(respBody); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); }); }); - }); - - describe('test form up#putFileWithoutKey', function() { - it('test form up#putFileWithoutKey', function(done) { - formUploader.putFileWithoutKey(uploadToken, testFilePath2, - putExtra, - function( - respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + // eslint-disable-next-line no-undef + describe('test form up#putFileWithoutKey', function () { + // eslint-disable-next-line no-undef + it('test form up#putFileWithoutKey', function () { + return formUploader.putFileWithoutKey(uploadToken, testFilePath_2, + putExtra, + function ( + respErr, + respBody) { + // console.log(respBody); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + }); + }); + }); + + describe('test form up#putFileWithFileType', function () { + it('test form up#putFileWithFileType IA', function () { + const key = 'storage_put_test_with_file_type' + Math.random(); + const putPolicy = new qiniu.rs.PutPolicy(Object.assign(options, { + fileType: 1 + })); + const uploadToken = putPolicy.uploadToken(mac); + return formUploader.putFile( + uploadToken, + key, + testFilePath_2, + putExtra, + function ( + respErr, + respBody + ) { + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + } + ); + }); + + it('test form up#putFileWithFileType Archive', function () { + const key = 'storage_put_test_with_file_type' + Math.random(); + const putPolicy = new qiniu.rs.PutPolicy(Object.assign(options, { + fileType: 2 + })); + const uploadToken = putPolicy.uploadToken(mac); + return formUploader.putFile( + uploadToken, + key, + testFilePath_2, + putExtra, + function ( + respErr, + respBody + ) { + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + } + ); + }); + + it('test form up#putFileWithFileType DeepArchive', function () { + const key = 'storage_put_test_with_file_type' + Math.random(); + const putPolicy = new qiniu.rs.PutPolicy(Object.assign(options, { + fileType: 3 + })); + const uploadToken = putPolicy.uploadToken(mac); + return formUploader.putFile( + uploadToken, + key, + testFilePath_2, + putExtra, + function ( + respErr, + respBody + ) { + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + } + ); + }); + }); + + // eslint-disable-next-line no-undef + describe('test form up#putFileWithParams', function () { + // eslint-disable-next-line no-undef + it('test form up#putFileWithMetadata', function (done) { + const key = 'storage_put_test_with_metadata' + Math.random(); + var putExtra = new qiniu.form_up.PutExtra(); + putExtra.metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + }; + formUploader.putFile(uploadToken, key, testFilePath_2, + putExtra, + function ( + respErr, + respBody) { + should.not.exist(respErr); + keysToDelete.push(respBody.key); + respBody.should.have.keys('key', 'hash'); + bucketManager.stat(bucket, key, function ( + err, + respBody, + respInfo + ) { + try { + should.not.exist(err); + respBody.should.have.keys('x-qn-meta'); + respBody['x-qn-meta'].name.should.eql('qiniu'); + respBody['x-qn-meta'].age.should.eql('18'); + done(); + } catch (e) { + done(e); + } + }); + }); + }); + + // eslint-disable-next-line no-undef + it('test form up#putFileWithCustomerData', function () { + const key = 'storage_put_test_with_customer_data'; + var putExtra = new qiniu.form_up.PutExtra(); + putExtra.params = { + 'x:location': 'shanghai', + 'x:price': 1500 + }; + return formUploader.putFile(uploadToken, key, testFilePath_2, + putExtra, + function ( + respErr, + respBody) { + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + respBody.should.have.keys('x:location', 'x:price'); + }); }); }); - }); }); diff --git a/test/github.png b/test/github.png deleted file mode 100644 index c7ce5c02..00000000 Binary files a/test/github.png and /dev/null differ diff --git a/test/httpc.test.js b/test/httpc.test.js new file mode 100644 index 00000000..72a92813 --- /dev/null +++ b/test/httpc.test.js @@ -0,0 +1,708 @@ +const should = require('should'); + +const fs = require('fs'); +const http = require('http'); +const os = require('os'); +const path = require('path'); + +const qiniu = require('../index'); + +const { + QUERY_REGION_HOST, + QUERY_REGION_BACKUP_HOSTS +} = qiniu.conf; + +const { + Middleware, + RetryDomainsMiddleware +} = qiniu.httpc.middleware; + +const { + Endpoint, + StaticEndpointsProvider, + SERVICE_NAME, + Region, + StaticRegionsProvider, + CachedRegionsProvider, + QueryRegionsProvider +} = qiniu.httpc; +const { + // eslint-disable-next-line camelcase + Zone_z0, + // eslint-disable-next-line camelcase + Zone_z1 +} = qiniu.zone; + +describe('test http module', function () { + const accessKey = process.env.QINIU_ACCESS_KEY; + // const secretKey = process.env.QINIU_SECRET_KEY; + const bucketName = process.env.QINIU_TEST_BUCKET; + + describe('test http ResponseWrapper', function () { + const { ResponseWrapper } = qiniu.httpc; + + it('needRetry', function () { + const cases = Array.from({ + length: 800 + }, (_, i) => { + if (i > 0 && i < 500) { + return { + code: i, + shouldRetry: false + }; + } + if ([ + 501, 509, 573, 579, 608, 612, 614, 616, 618, 630, 631, 632, 640, 701 + ].includes(i)) { + return { + code: i, + shouldRetry: false + }; + } + return { + code: i, + shouldRetry: true + }; + }); + cases.unshift({ + code: -1, + shouldRetry: true + }); + + const mockedResponseWrapper = new ResponseWrapper({ + data: [], + resp: { + statusCode: 200 + } + }); + + for (const item of cases) { + mockedResponseWrapper.resp.statusCode = item.code; + mockedResponseWrapper.needRetry().should.eql( + item.shouldRetry, + `${item.code} need${item.shouldRetry ? '' : ' NOT'} retry` + ); + } + }); + }); + + class OrderRecordMiddleware extends Middleware { + /** + * + * @param {string[]} record + * @param {string} label + */ + constructor (record, label) { + super(); + this.record = record; + this.label = label; + } + + /** + * + * @param {ReqOpts} request + * @param {function(ReqOpts):Promise} next + * @return {Promise} + */ + send (request, next) { + this.record.push(`bef_${this.label}${this.record.length}`); + return next(request).then((respWrapper) => { + this.record.push(`aft_${this.label}${this.record.length}`); + return respWrapper; + }); + } + } + + describe('test http middleware', function () { + let mockServer; + before(function (done) { + mockServer = http.createServer((req, res) => { + res.statusCode = 200; + res.setHeader('Content-Type', 'text/plain'); + res.end('Hello, Qiniu!\n'); + }); + mockServer.listen(9000, '127.0.0.1', done); + }); + + after(function () { + mockServer.close(); + }); + + it('test middleware', function (done) { + const recordList = []; + qiniu.rpc.qnHttpClient.sendRequest({ + url: 'http://127.0.0.1:9000/', + urllibOptions: { + method: 'GET', + followRedirect: true + }, + middlewares: [ + new OrderRecordMiddleware(recordList, 'A'), + new OrderRecordMiddleware(recordList, 'B') + ] + }) + .then(({ + _data, + resp + }) => { + recordList.should.eql(['bef_A0', 'bef_B1', 'aft_B2', 'aft_A3']); + should.equal(resp.statusCode, 200); + done(); + }) + .catch(err => { + done(err); + }); + }); + + it('test retry domains', function (done) { + const recordList = []; + qiniu.rpc.qnHttpClient.sendRequest({ + url: 'http://fake.nodesdk.qiniu.com/', + // url: 'https://qiniu.com/index.html', + urllibOptions: { + method: 'GET', + followRedirect: true + }, + middlewares: [ + new RetryDomainsMiddleware({ + backupDomains: [ + 'unavailable.pysdk.qiniu.com', + '127.0.0.1:9000' + ], + maxRetryTimes: 3 + }), + new OrderRecordMiddleware(recordList, 'A') + ] + }) + .then(({ + _data, + _resp + }) => { + recordList.should.eql([ + // fake.nodesdk.qiniu.com + 'bef_A0', + 'bef_A1', + 'bef_A2', + // unavailable.pysdk.qiniu.com + 'bef_A3', + 'bef_A4', + 'bef_A5', + // qiniu.com + 'bef_A6', + 'aft_A7' + ]); + done(); + }) + .catch(err => { + done(err); + }); + }); + + it('test retry domains fail fast', function (done) { + const recordList = []; + qiniu.rpc.qnHttpClient.sendRequest({ + url: 'http://fake.nodesdk.qiniu.com/', + // url: 'https://qiniu.com/index.html', + urllibOptions: { + method: 'GET', + followRedirect: true + }, + middlewares: [ + new RetryDomainsMiddleware({ + backupDomains: [ + 'unavailable.pysdk.qiniu.com', + '127.0.0.1:9000' + ], + retryCondition: () => false + }), + new OrderRecordMiddleware(recordList, 'A') + ] + }) + .then(({ + _data, + _resp + }) => { + done('this should not be ok'); + }) + .catch(_err => { + recordList.should.eql(['bef_A0']); + done(); + }); + }); + }); + + describe('test endpoint', function () { + it('test default options', function () { + const endpoint = new Endpoint('www.qiniu.com'); + should.equal(endpoint.getValue(), 'https://www.qiniu.com'); + should.equal(endpoint.getValue({ scheme: 'http' }), 'http://www.qiniu.com'); + }); + + it('test options', function () { + const endpoint = new Endpoint('www.qiniu.com', { + defaultScheme: 'http' + }); + should.equal(endpoint.getValue(), 'http://www.qiniu.com'); + should.equal(endpoint.getValue({ scheme: 'https' }), 'https://www.qiniu.com'); + }); + }); + + describe('test region', function () { + it('test default options', function () { + const region = new Region({ + regionId: 'z0' + }); + + should.equal(region.regionId, 'z0'); + should.equal(region.s3RegionId, 'z0'); + should.deepEqual( + Object.keys(region.services), + // use Object.values when min version of Node.js update to ≥ v7.5.0 + Object.keys(SERVICE_NAME).map(k => SERVICE_NAME[k]) + ); + should.ok(Date.now() - region.createTime.getTime() < 1000); + should.equal(region.ttl, 86400); + should.ok(region.isLive); + }); + + it('test options', function () { + const region = new Region({ + regionId: 'z0', + s3RegionId: 'z', + services: { + [SERVICE_NAME.UC]: [ + new Endpoint('fake-uc.qiniu.com') + ], + 'custom-service': [ + new Endpoint('custom-service.example.com') + ] + }, + createTime: new Date(Date.now() - 86400 * 1000), + ttl: 3600 + }); + + should.equal(region.regionId, 'z0'); + should.equal(region.s3RegionId, 'z'); + should.deepEqual( + Object.keys(region.services).sort(), + // use Object.values when min version of Node.js update to ≥ v7.5.0 + Object.keys(SERVICE_NAME).map(k => SERVICE_NAME[k]).concat(['custom-service']).sort() + ); + should.ok(new Date(Date.now() - 86400 * 1000).getTime() - region.createTime.getTime() < 1000); + should.equal(region.ttl, 3600); + should.ok(!region.isLive); + }); + + it('test fromZone', function () { + const regionZ0 = Region.fromZone(Zone_z0); + const upHosts = regionZ0.services[SERVICE_NAME.UP].map(endpoint => endpoint.host); + const srcUpHosts = upHosts.slice(0, Zone_z1.srcUpHosts.length); + const cdnUpHosts = upHosts.slice(Zone_z1.srcUpHosts.length); + const ioHosts = regionZ0.services[SERVICE_NAME.IO].map(endpoint => endpoint.host); + const rsHosts = regionZ0.services[SERVICE_NAME.RS].map(endpoint => endpoint.host); + const rsfHosts = regionZ0.services[SERVICE_NAME.RSF].map(endpoint => endpoint.host); + const apiHosts = regionZ0.services[SERVICE_NAME.API].map(endpoint => endpoint.host); + + should.equal(regionZ0.ttl, -1); + should.deepEqual(cdnUpHosts, Zone_z0.cdnUpHosts); + should.deepEqual(srcUpHosts, Zone_z0.srcUpHosts); + should.deepEqual(ioHosts, [Zone_z0.ioHost]); + should.deepEqual(rsHosts, [Zone_z0.rsHost]); + should.deepEqual(rsfHosts, [Zone_z0.rsfHost]); + should.deepEqual(apiHosts, [Zone_z0.apiHost]); + }); + + it('test fromZone with options', function () { + const regionZ1 = Region.fromZone(Zone_z1, { + ttl: 84600, + isPreferCdnHost: true + }); + const upHosts = regionZ1.services[SERVICE_NAME.UP].map(endpoint => endpoint.host); + const cdnUpHosts = upHosts.slice(0, Zone_z0.cdnUpHosts.length); + const srcUpHosts = upHosts.slice(Zone_z0.cdnUpHosts.length); + const ioHosts = regionZ1.services[SERVICE_NAME.IO].map(endpoint => endpoint.host); + const rsHosts = regionZ1.services[SERVICE_NAME.RS].map(endpoint => endpoint.host); + const rsfHosts = regionZ1.services[SERVICE_NAME.RSF].map(endpoint => endpoint.host); + const apiHosts = regionZ1.services[SERVICE_NAME.API].map(endpoint => endpoint.host); + + should.not.exist(regionZ1.regionId); + should.equal(regionZ1.ttl, 84600); + should.deepEqual(cdnUpHosts, Zone_z1.cdnUpHosts); + should.deepEqual(srcUpHosts, Zone_z1.srcUpHosts); + should.deepEqual(ioHosts, [Zone_z1.ioHost]); + should.deepEqual(rsHosts, [Zone_z1.rsHost]); + should.deepEqual(rsfHosts, [Zone_z1.rsfHost]); + should.deepEqual(apiHosts, [Zone_z1.apiHost]); + }); + + it('test fromRegionId', function () { + const regionZ0 = Region.fromRegionId('z0'); + + const servicesEndpointValues = {}; + // use Object.entries when min version of Node.js update to ≥ v7.5.0 + for (const serviceName of Object.keys(regionZ0.services)) { + const endpoints = regionZ0.services[serviceName]; + servicesEndpointValues[serviceName] = endpoints.map(e => e.getValue()); + } + + const expectedServicesEndpointValues = { + [SERVICE_NAME.UC]: [ + 'https://uc.qiniuapi.com' + ], + [SERVICE_NAME.UP]: [ + 'https://upload.qiniup.com', + 'https://up.qiniup.com', + 'https://up.qbox.me' + ], + [SERVICE_NAME.IO]: [ + 'https://iovip.qiniuio.com', + 'https://iovip.qbox.me' + ], + [SERVICE_NAME.RS]: [ + 'https://rs-z0.qiniuapi.com', + 'https://rs-z0.qbox.me' + ], + [SERVICE_NAME.RSF]: [ + 'https://rsf-z0.qiniuapi.com', + 'https://rsf-z0.qbox.me' + ], + [SERVICE_NAME.API]: [ + 'https://api-z0.qiniuapi.com', + 'https://api-z0.qbox.me' + ], + [SERVICE_NAME.S3]: [ + 'https://s3.z0.qiniucs.com' + ] + }; + + should.deepEqual(servicesEndpointValues, expectedServicesEndpointValues); + }); + + it('test fromRegionId with options', function () { + const regionZ1 = Region.fromRegionId( + 'z1', + { + s3RegionId: 'mock-z1', + ttl: -1, + createTime: new Date(0), + extendedServices: { + 'custom-service': [ + new Endpoint('custom-service.example.com') + ] + } + } + ); + + const servicesEndpointValues = {}; + // use Object.entries when min version of Node.js update to ≥ v7.5.0 + for (const serviceName of Object.keys(regionZ1.services)) { + const endpoints = regionZ1.services[serviceName]; + servicesEndpointValues[serviceName] = endpoints.map(e => e.getValue()); + } + + const expectedServicesEndpointValues = { + [SERVICE_NAME.UC]: [ + 'https://uc.qiniuapi.com' + ], + [SERVICE_NAME.UP]: [ + 'https://upload-z1.qiniup.com', + 'https://up-z1.qiniup.com', + 'https://up-z1.qbox.me' + ], + [SERVICE_NAME.IO]: [ + 'https://iovip-z1.qiniuio.com', + 'https://iovip-z1.qbox.me' + ], + [SERVICE_NAME.RS]: [ + 'https://rs-z1.qiniuapi.com', + 'https://rs-z1.qbox.me' + ], + [SERVICE_NAME.RSF]: [ + 'https://rsf-z1.qiniuapi.com', + 'https://rsf-z1.qbox.me' + ], + [SERVICE_NAME.API]: [ + 'https://api-z1.qiniuapi.com', + 'https://api-z1.qbox.me' + ], + [SERVICE_NAME.S3]: [ + 'https://s3.mock-z1.qiniucs.com' + ], + 'custom-service': [ + 'https://custom-service.example.com' + ] + }; + + should.deepEqual(servicesEndpointValues, expectedServicesEndpointValues); + should.equal(regionZ1.ttl, -1); + should.equal(regionZ1.createTime.getTime(), 0); + }); + }); + + describe('test endpoints provider', function () { + it('test StaticEndpointsProvider', function () { + const upEndpointsProvider = StaticEndpointsProvider.fromRegion( + Region.fromRegionId('z0'), + SERVICE_NAME.UP + ); + + return upEndpointsProvider.getEndpoints() + .then(endpoints => { + const endpointValues = endpoints.map(e => e.getValue()); + should.deepEqual(endpointValues, [ + 'https://upload.qiniup.com', + 'https://up.qiniup.com', + 'https://up.qbox.me' + ]); + }); + }); + }); + + describe('test regions provider', function () { + describe('test StaticRegionsProvider', function () { + it('test StaticRegionsProvider get', function () { + const staticRegionsProvider = new StaticRegionsProvider([ + Region.fromRegionId('z0'), + Region.fromRegionId('cn-east-2') + ]); + + return staticRegionsProvider.getRegions() + .then(regions => { + return regions.map(r => StaticEndpointsProvider.fromRegion(r, SERVICE_NAME.UP)); + }) + .then(endpointsProviders => { + return Promise.all(endpointsProviders.map(e => e.getEndpoints())); + }) + .then(regionsEndpoints => { + // use `Array.prototype.flat` if migrate to node v11.15 + const regionsEndpointValues = regionsEndpoints.map( + endpoints => + endpoints.map(e => e.getValue()) + ); + + should.deepEqual(regionsEndpointValues, [ + [ + 'https://upload.qiniup.com', + 'https://up.qiniup.com', + 'https://up.qbox.me' + ], + [ + 'https://upload-cn-east-2.qiniup.com', + 'https://up-cn-east-2.qiniup.com', + 'https://up-cn-east-2.qbox.me' + ] + ]); + }); + }); + }); + + describe('test CachedRegionsProvider', function () { + const cacheFilesToDelete = []; + const cacheKey = 'test-cache-key'; + + function getCachedRegionsProvider () { + const persistPath = path.join(process.cwd(), 'regions-cache-test' + cacheFilesToDelete.length + '.jsonl'); + cacheFilesToDelete.push(persistPath); + const result = new CachedRegionsProvider({ + cacheKey, + baseRegionsProvider: new StaticRegionsProvider([]), + persistPath + }); + result._memoCache = new Map(); + result.lastShrinkAt = new Date(0); + return result; + } + + after(function () { + cacheFilesToDelete.forEach(filePath => { + try { + fs.unlinkSync(filePath); + } catch (e) { + // ignore + } + }); + }); + + it('test CachedRegionsProvider getter', function () { + const cachedRegionsProvider = getCachedRegionsProvider(); + return cachedRegionsProvider.getRegions() + .then(regions => { + should.equal(regions.length, 0); + }); + }); + + it('test CachedRegionsProvider setter', function () { + const rZ0 = Region.fromRegionId('z0'); + const rZ1 = Region.fromRegionId('z1'); + const cachedRegionsProvider = getCachedRegionsProvider(); + + return cachedRegionsProvider.setRegions([ + rZ0, + rZ1 + ]) + .then(() => { + const content = fs.readFileSync(cachedRegionsProvider.persistPath); + const actual = JSON.parse(content.toString()); + should.deepEqual( + { + cacheKey: actual.cacheKey, + regions: actual.regions.map(r => r.regionId) + }, + { + cacheKey, + regions: [rZ0, rZ1].map(r => r.regionId) + } + ); + return cachedRegionsProvider.getRegions(); + }) + .then(regions => { + should.deepEqual(regions.map(r => r.regionId), ['z0', 'z1']); + }); + }); + + it('test CachedRegionsProvider getter with expired file cache', function () { + const cachedRegionsProvider = getCachedRegionsProvider(); + const rZ0 = Region.fromRegionId('z0'); + const mockCreateTime = new Date(); + mockCreateTime.setMinutes(1, 2, 3); + rZ0.createTime = mockCreateTime; + + const rZ0Expired = Region.fromRegionId('z0'); + rZ0Expired.createTime = new Date(0); + + fs.writeFileSync( + cachedRegionsProvider.persistPath, + JSON.stringify({ + cacheKey: cachedRegionsProvider.cacheKey, + regions: [rZ0Expired.persistInfo] + }) + ); + cachedRegionsProvider._memoCache.set(cachedRegionsProvider.cacheKey, [rZ0]); + return cachedRegionsProvider.getRegions() + .then(regions => { + should.equal(regions.length, 1); + const [actualRegion] = regions; + should.equal(actualRegion.createTime, mockCreateTime); + }); + }); + + it('test CachedRegionsProvider disable persist', function () { + const rZ0 = Region.fromRegionId('z0'); + const rZ1 = Region.fromRegionId('z1'); + const cachedRegionsProvider = getCachedRegionsProvider(); + const persistPath = cachedRegionsProvider.persistPath; + cachedRegionsProvider.persistPath = null; + + return cachedRegionsProvider.setRegions([ + rZ0, + rZ1 + ]) + .then(() => { + const persistFileExists = fs.existsSync(persistPath); + should.ok(!persistFileExists); + return cachedRegionsProvider.getRegions(); + }) + .then(regions => { + should.deepEqual(regions.map(r => r.regionId), ['z0', 'z1']); + }); + }); + + it('test CachedRegionsProvider with baseRegionsProvider', function () { + const cachedRegionsProvider = getCachedRegionsProvider(); + const expectRegions = [ + Region.fromRegionId('z0') + ]; + cachedRegionsProvider.baseRegionsProvider = new StaticRegionsProvider(expectRegions); + + return cachedRegionsProvider.getRegions() + .then(regions => { + should.deepEqual(regions.map(r => r.regionId), ['z0']); + should.equal(cachedRegionsProvider._memoCache.size, 1); + const content = fs.readFileSync(cachedRegionsProvider.persistPath); + const jsonl = content.toString().split(os.EOL).filter(l => l.length); + should.equal(jsonl.length, 1); + }); + }); + }); + + describe('test QueryRegionsProvider', function () { + /** + * @type {string[]} + */ + const queryRegionHosts = [ + QUERY_REGION_HOST + ] + .concat(QUERY_REGION_BACKUP_HOSTS); + + const queryRegionsProvider = new QueryRegionsProvider({ + accessKey: accessKey, + bucketName: bucketName, + endpointsProvider: new StaticEndpointsProvider( + queryRegionHosts.map(h => new Endpoint(h)) + ) + }); + + it('test QueryRegionsProvider getter', function () { + return queryRegionsProvider.getRegions() + .then(regions => { + should.ok(regions.length > 0, 'regions length should great than 0'); + }); + }); + + it('test QueryRegionsProvider error', function () { + const queryRegionsProvider = new QueryRegionsProvider({ + accessKey: 'fake', + bucketName: 'fake', + endpointsProvider: new StaticEndpointsProvider( + queryRegionHosts.map(h => new Endpoint(h)) + ) + }); + return queryRegionsProvider.getRegions() + .then( + () => { + should.not.exist('fake ak and fake bucket should be failed'); + }, + (err) => { + should.exist(err); + return Promise.resolve(); + } + ); + }); + + it('test QueryRegionsProvider with custom EndpointsProvider error', function () { + const queryRegionsProvider = new QueryRegionsProvider({ + accessKey: accessKey, + bucketName: bucketName, + endpointsProvider: new StaticEndpointsProvider([ + new Endpoint('fake-uc.csharp.qiniu.com') + ]) + }); + return queryRegionsProvider.getRegions() + .then( + () => { + should.not.exist('fake endpoint should be failed'); + }, + (err) => { + should.exist(err); + return Promise.resolve(); + } + ); + }); + + it('test QueryRegionsProvider with custom EndpointsProvider retried', function () { + const queryRegionsProvider = new QueryRegionsProvider({ + accessKey: accessKey, + bucketName: bucketName, + endpointsProvider: new StaticEndpointsProvider([ + new Endpoint('fake-uc.csharp.qiniu.com'), + new Endpoint(QUERY_REGION_HOST) + ]) + }); + return queryRegionsProvider.getRegions() + .then(regions => { + should.ok(regions.length > 0, 'regions length should great than 0'); + }); + }); + }); + }); +}); diff --git a/test/logo.png b/test/logo.png deleted file mode 100644 index a02df733..00000000 Binary files a/test/logo.png and /dev/null differ diff --git a/test/resume_up.test.js b/test/resume_up.test.js index 1f6d5405..a0ec4a97 100644 --- a/test/resume_up.test.js +++ b/test/resume_up.test.js @@ -1,92 +1,453 @@ +const os = require('os'); +const fs = require('fs'); const path = require('path'); const should = require('should'); -const assert = require('assert'); -const qiniu = require("../index.js"); -const proc = require("process"); -const fs = require("fs"); - -before(function(done) { - if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || ! - process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { - console.log('should run command `source test-env.sh` first\n'); - process.exit(0); - } - done(); +const qiniu = require('../index.js'); +const proc = require('process'); +const console = require('console'); +const crypto = require('crypto'); +const http = require('http'); +const Readable = require('stream').Readable; + +var testFilePath = path.join(os.tmpdir(), 'nodejs-sdk-test.bin'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + fs.createReadStream('/dev/urandom', { end: (1 << 20) * 10 }) + .pipe(fs.createWriteStream(testFilePath)) + .on('finish', done); }); +// file to upload + +// eslint-disable-next-line no-undef +describe('test resume up', function () { + this.timeout(0); + + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + var bucket = proc.env.QINIU_TEST_BUCKET; + var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + var config = new qiniu.conf.Config(); + config.useCdnDomain = true; + config.useHttpsDomain = true; + var bucketManager = new qiniu.rs.BucketManager(mac, config); + + // delete all the files uploaded + var keysToDelete = []; + + // eslint-disable-next-line no-undef + after(function (done) { + const deleteOps = []; + + keysToDelete.forEach(function (key) { + deleteOps.push(qiniu.rs.deleteOp(bucket, key)); + }); + + bucketManager.batch(deleteOps, function (respErr, respBody) { + respBody.forEach(function (ret, i) { + ret.code.should.be.eql( + 200, + JSON.stringify({ + key: keysToDelete[i], + ret: ret + }) + ); + }); + done(); + }); + }); -//file to upload -var imageFile = path.join(__dirname, 'logo.png'); + var options = { + scope: bucket + }; + var putPolicy = new qiniu.rs.PutPolicy(options); + putPolicy.returnBody = '{"key":$(key),"hash":$(etag),"fname":$(fname),"var_1":$(x:var_1),"var_2":$(x:var_2)}'; + var uploadToken = putPolicy.uploadToken(mac); + var resumeUploader = new qiniu.resume_up.ResumeUploader(config); -describe('test resume up', function() { - var accessKey = proc.env.QINIU_ACCESS_KEY; - var secretKey = proc.env.QINIU_SECRET_KEY; - var bucket = proc.env.QINIU_TEST_BUCKET; - var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); - var config = new qiniu.conf.Config(); - //config.useHttpsDomain = true; - config.zone = qiniu.zone.Zone_z0; - var bucketManager = new qiniu.rs.BucketManager(mac, config); + // eslint-disable-next-line no-undef + describe('test resume up#putFileWithoutKey', function () { + it('test resume up#putFileWithoutKey', function (done) { + var putExtra = new qiniu.resume_up.PutExtra(); + putExtra.params = { 'x:var_1': 'val_1', 'x:var_2': 'val_2' }; + putExtra.metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + }; + resumeUploader.putFileWithoutKey(uploadToken, testFilePath, + putExtra, + function ( + respErr, + respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + should(respBody['var_1']).eql('val_1'); + should(respBody['var_2']).eql('val_2'); + keysToDelete.push(respBody.key); - //delete all the files uploaded - var keysToDelete = []; + bucketManager.stat(bucket, respBody.key, function ( + err, + statRespBody, + respInfo + ) { + try { + should.not.exist(err); + statRespBody.should.have.keys('x-qn-meta'); + statRespBody['x-qn-meta'].name.should.eql('qiniu'); + statRespBody['x-qn-meta'].age.should.eql('18'); + } catch (e) { + done(e); + return; + } + done(); + }); + }); + }); + + it('test resume up#putFileWithoutKey_v2', function (done) { + var putExtra = new qiniu.resume_up.PutExtra(); + putExtra.partSize = 6 * 1024 * 1024 + putExtra.version = 'v2' + putExtra.params = { 'x:var_1': 'val_1', 'x:var_2': 'val_2' }; + putExtra.metadata = { + 'x-qn-meta-name': 'qiniu', + 'x-qn-meta-age': '18' + }; + resumeUploader.putFileWithoutKey(uploadToken, testFilePath, + putExtra, + function ( + respErr, + respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + should(respBody['var_1']).eql('val_1'); + should(respBody['var_2']).eql('val_2'); + if (keysToDelete.indexOf(respBody.key) === -1) { + keysToDelete.push(respBody.key); + } + bucketManager.stat(bucket, respBody.key, function ( + err, + statRespBody, + respInfo + ) { + try { + should.not.exist(err); + statRespBody.should.have.keys('x-qn-meta'); + statRespBody['x-qn-meta'].name.should.eql('qiniu'); + statRespBody['x-qn-meta'].age.should.eql('18'); + } catch (e) { + done(e); + return; + } + done(); + }); + }); + }); - after(function(done) { - var deleteOps = []; - keysToDelete.forEach(function(key) { - deleteOps.push(qiniu.rs.deleteOp(bucket, key)); }); - bucketManager.batch(deleteOps, function(respErr, respBody, respInfo) { - //console.log(respBody); - respBody.forEach(function(ret) { - ret.should.eql({ - code: 200 + // eslint-disable-next-line no-undef + describe('test resume up#putFile', function () { + it('test resume up#putFile', function (done) { + const putExtra = new qiniu.resume_up.PutExtra(); + putExtra.mimeType = 'application/json'; + const key = 'storage_putFile_test' + Math.floor(Math.random() * 1000); + const domain = proc.env.QINIU_TEST_DOMAIN; + resumeUploader.putFile(uploadToken, key, testFilePath, putExtra, + function (respErr, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + + const localFileMd5Promise = new Promise(function (resolve) { + const md5 = crypto.createHash('md5'); + const stream = fs.createReadStream(testFilePath); + stream.on('data', function (data) { + md5.update(data); + }); + stream.on('end', function () { + resolve(md5.digest('hex')); + }); + }); + const remoteFileMd5Promise = new Promise(function (resolve) { + http.get('http://' + domain + '/' + key, function (response) { + response.statusCode.should.eql(200); + response.headers['content-type'].should.eql('application/json'); + const md5 = crypto.createHash('md5'); + response.on('data', function (data) { + md5.update(data); + }); + response.on('end', function () { + resolve(md5.digest('hex')); + }); + }); + }); + + Promise.all([localFileMd5Promise, remoteFileMd5Promise]) + .then(function ([expectedMd5, actualMd5]) { + try { + actualMd5.should.eql(expectedMd5); + } catch (e) { + done(e); + return; + } + done(); + }); + }); + }); + + it('test resume up#putFile_v2', function (done) { + const putExtra = new qiniu.resume_up.PutExtra(); + const key = 'storage_putFile_test_v2' + Math.floor(Math.random() * 1000); + const domain = proc.env.QINIU_TEST_DOMAIN; + putExtra.partSize = 6 * 1024 * 1024; + putExtra.version = 'v2'; + putExtra.mimeType = 'application/x-www-form-urlencoded'; + resumeUploader.putFile(uploadToken, key, testFilePath, putExtra, + function (respErr, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + + const localFileMd5Promise = new Promise(function (resolve) { + const md5 = crypto.createHash('md5'); + const stream = fs.createReadStream(testFilePath); + stream.on('data', function (data) { + md5.update(data); + }); + stream.on('end', function () { + resolve(md5.digest('hex')); + }); + }); + const remoteFileMd5Promise = new Promise(function (resolve) { + http.get('http://' + domain + '/' + key, function (response) { + response.statusCode.should.eql(200); + response.headers['content-type'].should.eql('application/x-www-form-urlencoded'); + const md5 = crypto.createHash('md5'); + response.on('data', function (data) { + md5.update(data); + }); + response.on('end', function () { + resolve(md5.digest('hex')); + }); + }); + }); + + Promise.all([localFileMd5Promise, remoteFileMd5Promise]) + .then(function ([expectedMd5, actualMd5]) { + try { + actualMd5.should.eql(expectedMd5); + } catch (e) { + done(e); + return; + } + done(); + }); + }); }); - }); - done(); }); - }); - - var options = { - scope: bucket, - } - var putPolicy = new qiniu.rs.PutPolicy(options); - var uploadToken = putPolicy.uploadToken(mac); - var config = new qiniu.conf.Config(); - config.zone = qiniu.zone.Zone_z0; - var resumeUploader = new qiniu.resume_up.ResumeUploader(config); - var putExtra = new qiniu.resume_up.PutExtra(); - - describe('test resume up#putFileWithoutKey', function() { - it('test resume up#putFileWithoutKey', function(done) { - resumeUploader.putFileWithoutKey(uploadToken, imageFile, - putExtra, - function( - respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + describe('test resume up#putStream', function () { + it('test resume up#putStream', function (done) { + var putExtra = new qiniu.resume_up.PutExtra(); + putExtra.mimeType = 'application/x-www-form-urlencoded'; + var key = 'storage_putStream_test' + Math.random(1000); + var stream = new Readable(); + var domain = proc.env.QINIU_TEST_DOMAIN; + var blkSize = 1024 * 1024; + var blkCnt = 9; + var expectedMd5Crypto = crypto.createHash('md5'); + for (var i = 0; i < blkCnt; i++) { + var bytes = crypto.randomBytes(blkSize); + stream.push(bytes); + expectedMd5Crypto.update(bytes); + } + stream.push(null); + var expectedMd5 = expectedMd5Crypto.digest('hex'); + resumeUploader.putStream(uploadToken, key, stream, blkCnt * blkSize, putExtra, + function (respErr, respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + + http.get("http://" + domain + "/" + key, function (response) { + response.statusCode.should.eql(200); + response.headers['content-type'].should.eql('application/x-www-form-urlencoded'); + { + var actualMd5Crypto = crypto.createHash('md5'); + response.on('data', function (data) { + actualMd5Crypto.update(data); + }); + response.on('end', function () { + try { + var actualMd5 = actualMd5Crypto.digest('hex'); + should(actualMd5).eql(expectedMd5); + } catch (e) { + done(e); + return; + } + done(); + }); + } + }); + }); + }); + + it('test resume up#putStream_v2', function (done) { + var putExtra = new qiniu.resume_up.PutExtra(); + putExtra.mimeType = 'application/xml'; + var key = 'storage_putStream_test_v2' + Math.random(1000); + var stream = new Readable(); + var domain = proc.env.QINIU_TEST_DOMAIN; + var blkSize = 1024 * 1024; + var blkCnt = 9; + var expectedMd5Crypto = crypto.createHash('md5'); + for (var i = 0; i < blkCnt; i++) { + var bytes = crypto.randomBytes(blkSize); + stream.push(bytes); + expectedMd5Crypto.update(bytes); + } + stream.push(null); + var expectedMd5 = expectedMd5Crypto.digest('hex'); + putExtra.partSize = 6 * 1024 * 1024 + putExtra.version = 'v2' + resumeUploader.putStream(uploadToken, key, stream, blkCnt * blkSize, putExtra, + function ( + respErr, + respBody, respInfo) { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + + http.get("http://" + domain + "/" + key, function (response) { + response.statusCode.should.eql(200); + response.headers['content-type'].should.eql('application/xml'); + { + var actualMd5Crypto = crypto.createHash('md5'); + response.on('data', function (data) { + actualMd5Crypto.update(data); + }); + response.on('end', function () { + try { + var actualMd5 = actualMd5Crypto.digest('hex'); + should(actualMd5).eql(expectedMd5); + } catch (e) { + done(e); + return; + } + done(); + }); + } + }); + }); }); }); - }); - - describe('test resume up#putFile', function() { - it('test resume up#putFile', function(done) { - var key = 'storage_putFile_test' + Math.random(1000); - resumeUploader.putFile(uploadToken, key, imageFile, putExtra, - function( - respErr, - respBody, respInfo) { - //console.log(respBody); - should.not.exist(respErr); - respBody.should.have.keys('key', 'hash'); - keysToDelete.push(respBody.key); - done(); + + describe('test resume up#putStream resume', function () { + it('test resume up#putStream resume', function (done) { + var putExtra = new qiniu.resume_up.PutExtra(); + config.zone = null; + var key = 'storage_putStream_resume_test' + Math.random(1000); + var stream = new Readable(); + var blkSize = 1024 * 1024; + var blkCnt = 4; + for (var i = 0; i < blkCnt; i++) { + stream.push(crypto.randomBytes(blkSize)); + } + stream.push(null); + var tmpfile = path.join(os.tmpdir(), '/resume_file'); + fs.writeFileSync(tmpfile, ''); + putExtra.resumeRecordFile = tmpfile; + putExtra.progressCallback = function (len, total) { + if (len === total) { + var content = fs.readFileSync(tmpfile); + var data = JSON.parse(content); + data.upDomains.should.not.empty(); + data.parts.forEach(function (item) { + item.should.have.keys('ctx', 'expired_at', 'crc32'); + }); + } + }; + resumeUploader.putStream(uploadToken, key, stream, blkCnt * blkSize, putExtra, + function ( + respErr, + respBody, respInfo + ) { + try { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + } catch (e) { + done(e); + return; + } + done(); + }); + }); + + it('test resume up#putStream resume_v2', function (done) { + var putExtra = new qiniu.resume_up.PutExtra(); + config.zone = null; + var num = 0; + var blkSize = 1024 * 1024; + var blkCnt = [2, 4, 4.1, 6, 10]; + var tmpfile = path.join(os.tmpdir(), '/resume_file'); + fs.writeFileSync(tmpfile, ''); + putExtra.resumeRecordFile = tmpfile; + putExtra.partSize = 4 * 1024 * 1024; + putExtra.version = 'v2'; + putExtra.progressCallback = function (len, total) { + if (len === total) { + var content = fs.readFileSync(tmpfile); + var data = JSON.parse(content); + data.etags.forEach(function (item) { + item.should.have.keys('etag', 'partNumber'); + }); + } + }; + blkCnt.forEach(function (i) { + var stream = new Readable(); + for (var j = 0; j < i; j++) { + stream.push(crypto.randomBytes(blkSize)); + } + if (i === +i && i !== (i | 0)) { + stream.push('0f'); + } + stream.push(null); + var key = 'storage_putStream_resume_test_v2' + Math.random(1000); + resumeUploader.putStream(uploadToken, key, stream, i * blkSize, putExtra, + function ( + respErr, + respBody, + respInfo + ) { + try { + console.log(respBody, respInfo); + should.not.exist(respErr); + respBody.should.have.keys('key', 'hash'); + keysToDelete.push(respBody.key); + num++; + } catch (e) { + done(e); + return; + } + if (num === blkCnt.length) { + done(); + } + }); + }); }); }); - }); }); diff --git a/test/rs.test.js b/test/rs.test.js index a1fe2bc3..ecc5e0f8 100644 --- a/test/rs.test.js +++ b/test/rs.test.js @@ -1,143 +1,1013 @@ const should = require('should'); const assert = require('assert'); -const qiniu = require("../index.js"); -const proc = require("process"); - -before(function(done) { - if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || ! - process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { - console.log('should run command `source test-env.sh` first\n'); - process.exit(0); - } - done(); +const qiniu = require('../index.js'); +const proc = require('process'); +const urllib = require('urllib'); +const console = require('console'); +const mockDate = require('mockdate'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + done(); }); -describe('test start bucket manager', function() { - var accessKey = proc.env.QINIU_ACCESS_KEY; - var secretKey = proc.env.QINIU_SECRET_KEY; - var srcBucket = proc.env.QINIU_TEST_BUCKET; - var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); - var config = new qiniu.conf.Config(); - //config.useHttpsDomain = true; - config.zone = qiniu.zone.Zone_z0; - var bucketManager = new qiniu.rs.BucketManager(mac, config); - //test stat - describe('test stat', function() { - it('test stat', function(done) { - var bucket = srcBucket; - var key = 'qiniu.mp4'; - bucketManager.stat(bucket, key, function(err, respBody, - respInfo) { - //console.log(respBody); - should.not.exist(err); - respBody.should.have.keys('hash', 'fsize', 'mimeType', - 'putTime', 'type'); - done(); - }); - }); - }); - - //test copy and move and delete - describe('test copy', function() { - it('test copy', function(done) { - var destBucket = srcBucket; - var srcKey = 'qiniu.mp4'; - var destKey = 'qiniu_copy.mp4'; - var options = { - force: true, - } - bucketManager.copy(srcBucket, srcKey, destBucket, destKey, - options, - function(err, respBody, respInfo) { - //console.log(respBody); - should.not.exist(err); - assert.equal(respInfo.statusCode, 200); - done(); - - //test move - describe('test move', function() { - var moveDestKey = 'qiniu_move.mp4'; - it('test move', function(done1) { - bucketManager.move(destBucket, destKey, - destBucket, moveDestKey, options, - function(err1, ret1, info1) { - should.not.exist(err1); - assert.equal(info1.statusCode, 200); - done1(); - - //test delete - describe('test delete', function() { - it('test delete', function( - done2) { - bucketManager.delete( - destBucket, - moveDestKey, - function(err2, ret2, - info2) { - should.not.exist( - err2); - assert.equal(info2.statusCode, - 200); - done2(); +// eslint-disable-next-line no-undef +describe('test start bucket manager', function () { + this.timeout(0); + + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + var srcBucket = proc.env.QINIU_TEST_BUCKET; + var domain = proc.env.QINIU_TEST_DOMAIN; + var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + var config = new qiniu.conf.Config(); + // config.useCdnDomain = true; + config.useHttpsDomain = true; + var bucketManager = new qiniu.rs.BucketManager(mac, config); + + const keysToDeleteAfter = []; + after(function (done) { + const deleteOps = []; + keysToDeleteAfter.forEach(function (key) { + deleteOps.push(qiniu.rs.deleteOp(srcBucket, key)); + }); + + deleteOps.length && bucketManager.batch(deleteOps, function (respErr, respBody) { + respBody.forEach(function (ret, i) { + ret.code.should.be.eql( + 200, + JSON.stringify({ + key: keysToDeleteAfter[i], + ret: ret + }) + ); + }); + done(); + }); + }); + + // TODO: using this method to wrapper all operation. done tests: + // - restoreAr + // - setObjectLifecycle + function testObjectOperationWrapper (destBucket, destObjectKey, callback) { + const srcKey = 'qiniu.mp4'; + const destKey = destObjectKey + Math.random(); + const options = { + force: true + }; + bucketManager.copy(srcBucket, srcKey, destBucket, destKey, options, + function (err, respBody, respInfo) { + // console.log(respBody); + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + keysToDeleteAfter.push(destKey); + callback(destKey); + }); + } + // test stat + // eslint-disable-next-line no-undef + describe('test stat', function () { + // eslint-disable-next-line no-undef + it('test stat', function (done) { + var bucket = srcBucket; + var key = 'qiniu.mp4'; + bucketManager.stat(bucket, key, function (err, respBody, + respInfo) { + console.log(respBody, respInfo); + should.not.exist(err); + respBody.should.have.keys('hash', 'fsize', 'mimeType', + 'putTime', 'type'); + done(); + }); + }); + }); + + describe('test privateDownloadUrl', function () { + it('test privateDownloadUrl', function (done) { + var key = 'test_file'; + var url = bucketManager.privateDownloadUrl('http://' + domain, key, 20); + urllib.request(url, function (err, respBody, respInfo) { + console.log(respBody.toString(), respInfo); + should.not.exist(err); + should.equal(respInfo.status, 200); + done(); + }); + }); + }); + + // test copy and move and delete + // eslint-disable-next-line no-undef + describe('test copy', function () { + // eslint-disable-next-line no-undef + it('test copy', function (done) { + var destBucket = srcBucket; + var srcKey = 'qiniu.mp4'; + var destKey = 'qiniu_copy.mp4'; + var options = { + force: true + }; + bucketManager.copy(srcBucket, srcKey, destBucket, destKey, + options, + function (err, respBody, respInfo) { + // console.log(respBody); + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + + // test move + // eslint-disable-next-line no-undef + describe('test move', function () { + var moveDestKey = 'qiniu_move.mp4'; + // eslint-disable-next-line no-undef + it('test move', function (done1) { + bucketManager.move(destBucket, destKey, + destBucket, moveDestKey, options, + function (err1, ret1, info1) { + should.not.exist(err1); + assert.strictEqual(info1.statusCode, 200); + done1(); + + // test delete + // eslint-disable-next-line no-undef + describe('test delete', function () { + // eslint-disable-next-line no-undef + it('test delete', function ( + done2) { + bucketManager.delete( + destBucket, + moveDestKey, + function (err2, ret2, + info2) { + should.not.exist( + err2); + assert.strictEqual(info2.statusCode, + 200); + done2(); + }); + }); + }); + }); + }); + }); + }); + }); + }); + + // test copy and deleteAfterDays + describe('test copy', function () { + it('test copy', function (done) { + var destBucket = srcBucket; + var srcKey = 'qiniu.mp4'; + var destKey = 'qiniu_delete_after_days.mp4'; + var options = { + force: true + }; + bucketManager.copy(srcBucket, srcKey, destBucket, destKey, options, + function (err, respBody, respInfo) { + // console.log(respBody); + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + + // test deleteAfterDays + describe('test deleteAfterDays', function () { + it('test deleteAfterDays', function (done1) { + bucketManager.deleteAfterDays(destBucket, destKey, 1, + function (err1, ret1, info1) { + should.not.exist(err1); + assert.strictEqual(info1.statusCode, 200); + done1(); + }); }); }); - }); }); + }); + }); + + // eslint-disable-next-line no-undef + describe('test fetch', function () { + // eslint-disable-next-line no-undef + it('test fetch', function (done) { + const resUrl = 'http://devtools.qiniu.com/qiniu.png'; + const bucket = srcBucket; + const key = 'qiniu.png'; + + bucketManager.fetch(resUrl, bucket, key, + function (err, respBody, respInfo) { + should.not.exist(err, respInfo); + respBody.should.have.keys( + 'hash', + 'fsize', + 'mimeType', + 'key' + ); + done(); + } + ); + }); + }); + + // eslint-disable-next-line no-undef + describe('test changeMime', function () { + // eslint-disable-next-line no-undef + it('test changeMime', function (done) { + var key = 'test_file'; + var bucket = srcBucket; + + bucketManager.changeMime(bucket, key, 'text/html', + function (err, respBody, respInfo) { + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + } + ); + }); + }); + + // eslint-disable-next-line no-undef + describe('test changeHeaders', function () { + // eslint-disable-next-line no-undef + it('test changeHeaders', function (done) { + var key = 'test_file'; + var bucket = srcBucket; + + bucketManager.changeHeaders(bucket, key, { + 'Content-Type': 'text/plain', + 'Last-Modified': 'Wed, 21 Oct 2015 07:28:00 GMT', + 'x-qn-test-custom-header': '0' + }, function (err, respBody, respInfo) { + console.log(respInfo); + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + }); + + // stat file and changeType + describe('test changeType', function () { + it('test changeType', function (done) { + var key = 'test_file'; + var bucket = srcBucket; + bucketManager.stat(bucket, key, function (e, res, info) { + should.not.exist(e); + assert.strictEqual(info.statusCode, 200); + var type = res.type === 1 ? 0 : 1; + bucketManager.changeType(bucket, key, type, + function (err, respBody, respInfo) { + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + } + ); + }); + }); + }); + + describe('test updateObjectStatus', function () { + it('test updateObjectStatus disable', function (done) { + var key = 'test_file'; + var bucket = srcBucket; + bucketManager.updateObjectStatus(bucket, key, 1, function (err, respBody, respInfo) { + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + + it('test updateObjectStatus enable', function (done) { + var key = 'test_file'; + var bucket = srcBucket; + bucketManager.updateObjectStatus(bucket, key, 0, function (err, respBody, respInfo) { + should.not.exist(err); + assert.strictEqual(respInfo.statusCode, 200); + done(); + }); + }); + }); + + describe('test listBucket', function () { + it('test listBucket', function (done) { + bucketManager.listBucket(function (err, + respBody, respInfo) { + should.not.exist(err); + console.log(JSON.stringify(respBody) + '\n'); + console.log(JSON.stringify(respInfo)); + respBody.should.containEql(srcBucket); + done(); + }); + }); + }); + + // eslint-disable-next-line no-undef + describe('test bucketinfo', function () { + // eslint-disable-next-line no-undef + it('test bucketinfo', function (done) { + var bucket = srcBucket; + + bucketManager.getBucketInfo(bucket, function (err, + respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test listPrefix', function () { + it('test listPrefix', function (done) { + var bucket = srcBucket; + + bucketManager.listPrefix(bucket, { + prefix: 'test' + }, function (err, respBody, respInfo) { + should.not.exist(err); + console.log(JSON.stringify(respBody) + '\n'); + console.log(JSON.stringify(respInfo)); + respBody.should.have.keys('items'); + respBody.items.forEach(function (item) { + item.should.have.keys('key', 'hash'); + item.key.should.startWith('test'); + }); + done(); }); - }); }); }); - }); - describe('test fetch', function() { - it('test fetch', function(done) { - var resUrl = 'http://devtools.qiniu.com/qiniu.png'; - var bucket = srcBucket; - var key = "qiniu.png"; + describe('test listPrefixV2', function () { + it('test listPrefixV2', function (done) { + var bucket = srcBucket; - bucketManager.fetch(resUrl, bucket, key, function(err, - respBody, - respInfo) { - should.not.exist(err); - respBody.should.have.keys('hash', 'fsize', 'mimeType', - 'key'); - done(); - }); + bucketManager.listPrefixV2(bucket, { + prefix: 'test' + }, function (_err, respBody, respInfo) { + // the irregular data return from Server that Cannot be converted by urllib to JSON Object + // so err !=null and you can judge respBody==null or err.res.statusCode==200 + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); }); - }); - describe('test changeMime', function() { - it('test changeMime', function(done) { - var key = "test_file"; - var bucket = srcBucket; + // 空间生命周期 + describe('test lifeRule', function () { + const bucket = srcBucket; + const ruleName = 'test_rule_name'; - bucketManager.changeMime(bucket, key, "text/html", - function (err, respBody, respInfo) { - should.not.exist(err); - assert.equal(respInfo.statusCode, 200); - done(); + function testGet (expectItem, nextCall, otherRespInfo) { + bucketManager.getBucketLifecycleRule( + bucket, + function ( + err, + respBody, + respInfo + ) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + if (!expectItem && !respBody) { + nextCall(); + return; + } + const actualItem = respBody.find(function (item) { + return item.name === ruleName; + }); + if (!expectItem) { + should.not.exist(actualItem); + nextCall(); + return; + } + should.exist(actualItem, JSON.stringify({ + respInfo: respInfo, + otherRespInfo: otherRespInfo + })); + actualItem.should.have.properties(expectItem); + nextCall(); + } + ); } - ); - }); - }); - - describe('test changeHeaders', function() { - it('test changeHeaders', function(done) { - var key = "test_file"; - var bucket = srcBucket; - - bucketManager.changeHeaders(bucket, key, { - 'Content-Type': 'text/plain', - 'Last-Modified': 'Wed, 21 Oct 2015 07:28:00 GMT', - 'x-qn-test-custom-header': '0', - }, - function (err, respBody, respInfo) { - console.log(respInfo); - should.not.exist(err); - assert.equal(respInfo.statusCode, 200); - done(); + + before(function (done) { + bucketManager.deleteBucketLifecycleRule(bucket, ruleName, done); + }); + + it('test lifeRule put', function (done) { + const options = { + name: ruleName, + prefix: 'test', + to_line_after_days: 30, + to_archive_ir_after_days: 35, + to_archive_after_days: 40, + to_deep_archive_after_days: 50, + delete_after_days: 65 + }; + bucketManager.putBucketLifecycleRule( + bucket, + options, + function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + testGet({ + prefix: 'test', + to_line_after_days: 30, + to_archive_ir_after_days: 35, + to_archive_after_days: 40, + to_deep_archive_after_days: 50, + delete_after_days: 65, + history_delete_after_days: 0, + history_to_line_after_days: 0 + }, done, respInfo); + } + ); + }); + + it('test lifeRule update', function (done) { + const options = { + name: ruleName, + prefix: 'update_prefix', + to_line_after_days: 30, + to_archive_ir_after_days: 40, + to_archive_after_days: 50, + to_deep_archive_after_days: 60, + delete_after_days: 65 + }; + bucketManager.updateBucketLifecycleRule( + bucket, + options, + function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + + testGet({ + prefix: 'update_prefix', + to_line_after_days: 30, + to_archive_ir_after_days: 40, + to_archive_after_days: 50, + to_deep_archive_after_days: 60, + delete_after_days: 65, + history_delete_after_days: 0, + history_to_line_after_days: 0 + }, done, respInfo); + } + ); + }); + + it('test lifeRule delete', function (done) { + bucketManager.deleteBucketLifecycleRule( + bucket, + ruleName, + function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + testGet(null, done, respInfo); + } + ); + }); + }); + + describe('test object lifecycle', function () { + const bucket = srcBucket; + + it('test setObjectLifeCycle', function (done) { + testObjectOperationWrapper(bucket, 'test_set_object_lifecycle', function (key) { + bucketManager.setObjectLifeCycle( + bucket, + key, + { + toIaAfterDays: 10, + toArchiveIRAfterDays: 15, + toArchiveAfterDays: 20, + toDeepArchiveAfterDays: 30, + deleteAfterDays: 40 + }, + function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + done(); + } + ); + }); + }); + + it('test setObjectLifeCycle with cond', function (done) { + testObjectOperationWrapper(bucket, 'test_set_object_lifecycle_cond', function (key) { + bucketManager.stat(bucket, key, function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + const { hash } = respBody; + + bucketManager.setObjectLifeCycle( + bucket, + key, + { + toIaAfterDays: 10, + toArchiveIRAfterDays: 15, + toArchiveAfterDays: 20, + toDeepArchiveAfterDays: 30, + deleteAfterDays: 40, + cond: { + hash: hash + } + }, + function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + done(); + } + ); + }); + }); + }); + }); + + describe('test events', function () { + const bucket = srcBucket; + const eventName = 'event_test'; + + before(function (done) { + bucketManager.deleteBucketEvent( + bucket, + eventName, + function () { + done(); + } + ); + }); + + it('test addEvents', function (done) { + const options = { + name: eventName, + event: 'mkfile', + callbackURL: 'http://node.ijemy.com/qncback' + }; + bucketManager.putBucketEvent( + bucket, + options, + function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + } + ); + }); + + it('test updateEvents', function (done) { + const options = { + name: eventName, + event: 'copy', + callbackURL: 'http://node.ijemy.com/qncback' + }; + bucketManager.updateBucketEvent( + bucket, + options, + function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + } + ); + }); + + it('test getEvents', function (done) { + bucketManager.getBucketEvent( + bucket, + function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + } + ); + }); + + it('test deleteEvents', function (done) { + bucketManager.deleteBucketEvent( + bucket, + eventName, + function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + } + ); + }); + }); + + describe('test referAntiLeech', function () { + describe('test referAntiLeech', function () { + var options = { + mode: 1, + norefer: 0, + pattern: '*.iorange.vip' + }; + var bucket = srcBucket; + it('test referAntiLeech', function (done) { + bucketManager.putReferAntiLeech(bucket, options, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + }); + + describe('test corsRules', function () { + var bucket = srcBucket; + describe('test putCorsRules', function () { + it('test putCorsRules', function (done) { + var body = []; + var req01 = { + allowed_origin: ['http://www.test1.com'], + allowed_method: ['GET', 'POST'] + }; + var req02 = { + allowed_origin: ['http://www.test2.com'], + allowed_method: ['GET', 'POST', 'HEAD'], + allowed_header: ['testheader', 'Content-Type'], + exposed_header: ['test1', 'test2'], + max_age: 20 + }; + body[0] = req01; + body[1] = req02; + + bucketManager.putCorsRules(bucket, body, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test getCorsRules', function () { + it('test getCorsRules', function (done) { + bucketManager.getCorsRules(bucket, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + }); + // + // describe('test mirrorConfig', function() { + // describe('test getMirrorConfig', function() { + // var bucket = srcBucket; + // it('test getMirrorConfig', function(done) { + // var body = { + // "bucket":bucket, + // }; + // bucketManager.getBucketSourceConfig(body, function(err, respBody, respInfo) { + // should.not.exist(err); + // console.log(JSON.stringify(respBody) + "\n"); + // console.log(JSON.stringify(respInfo)); + // done(); + // }); + // }); + // }); + // }); + + describe('test accessMode', function () { + var bucket = srcBucket; + it('test accessMode', function (done) { + var mode = 0; + bucketManager.putBucketAccessStyleMode(bucket, mode, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test restoreAr', function () { + const bucket = srcBucket; + + function changeType (key, type, callback) { + bucketManager.changeType( + bucket, + key, + type, + function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + callback(); + } + ); } - ); + + it('test restoreAr Archive', function (done) { + testObjectOperationWrapper(bucket, 'test_restore_ar_archive', function (key) { + // change file type to Archive + changeType(key, 2, function () { + const freezeAfterDays = 1; + const entry = bucket + (key ? ':' + key : ''); + bucketManager.restoreAr(entry, freezeAfterDays, function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + }); + + it('test restoreAr DeepArchive', function (done) { + testObjectOperationWrapper(bucket, 'test_restore_ar_deep_archive', function (key) { + // change file type to DeepArchive + changeType(key, 3, function () { + const freezeAfterDays = 2; + const entry = bucket + (key ? ':' + key : ''); + bucketManager.restoreAr(entry, freezeAfterDays, function (err, respBody, respInfo) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + }); + }); + + describe('test putBucketMaxAge', function () { + var bucket = srcBucket; + it('test putBucketMaxAge', function (done) { + var options = { + maxAge: 0 + }; + bucketManager.putBucketMaxAge(bucket, options, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test putBucketAccessMode', function () { + var bucket = srcBucket; + it('test putBucketAccessMode', function (done) { + var options = { + private: 0 + }; + bucketManager.putBucketAccessMode(bucket, options, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test bucketQuota', function () { + var bucket = srcBucket; + describe('test putBucketQuota', function () { + it('test putBucketQuota', function (done) { + var options = { + size: 10, + count: 10 + }; + bucketManager.putBucketQuota(bucket, options, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + it('test cancel putBucketQuota', function (done) { + var options = { + size: -1, + count: -1 + }; + bucketManager.putBucketQuota(bucket, options, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + describe('test getBucketQuota', function () { + it('test getBucketQuota', function (done) { + bucketManager.getBucketQuota(bucket, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + }); + + describe('test listBucketDomains', function () { + var bucket = srcBucket; + it('test listBucketDomains', function (done) { + bucketManager.listBucketDomains(bucket, function (err, respBody, respInfo) { + should.not.exist(err); + should.equal(respInfo.status, 200, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test invalid X-Qiniu-Date', function () { + beforeEach(function () { + mockDate.set(new Date(0)); + }); + + afterEach(function () { + delete process.env.DISABLE_QINIU_TIMESTAMP_SIGNATURE; + }); + + after(function () { + mockDate.reset(); + }); + + it('test invalid X-Qiniu-Date expect 403', function (done) { + const bucket = srcBucket; + const key = 'qiniu.mp4'; + bucketManager.stat(bucket, key, function ( + err, + respBody, + respInfo + ) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(403, JSON.stringify(respInfo)); + done(); + }); + }); + + it('test invalid X-Qiniu-Date expect 200 by disable date sign', function (done) { + const mac = new qiniu.auth.digest.Mac( + accessKey, + secretKey, + { disableQiniuTimestampSignature: true } + ); + const config = new qiniu.conf.Config({ + useHttpsDomain: true + }); + const bucketManager = new qiniu.rs.BucketManager(mac, config); + + const bucket = srcBucket; + const key = 'qiniu.mp4'; + bucketManager.stat(bucket, key, function ( + err, + respBody, + respInfo + ) { + should.not.exist(err, JSON.stringify(respInfo)); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + respBody.should.have.keys( + 'hash', + 'fsize', + 'mimeType', + 'putTime', + 'type' + ); + done(); + }); + }); + + it('test invalid X-Qiniu-Date env expect 200', function (done) { + process.env.DISABLE_QINIU_TIMESTAMP_SIGNATURE = 'true'; + const bucket = srcBucket; + const key = 'qiniu.mp4'; + bucketManager.stat(bucket, key, function ( + err, + respBody, + respInfo + ) { + should.not.exist(err, JSON.stringify(respInfo)); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + respBody.should.have.keys( + 'hash', + 'fsize', + 'mimeType', + 'putTime', + 'type' + ); + done(); + }); + }); + it('test invalid X-Qiniu-Date env be ignored expect 403', function (done) { + process.env.DISABLE_QINIU_TIMESTAMP_SIGNATURE = 'true'; + const mac = new qiniu.auth.digest.Mac( + accessKey, + secretKey, + { disableQiniuTimestampSignature: false } + ); + const config = new qiniu.conf.Config({ + useHttpsDomain: true + }); + const bucketManager = new qiniu.rs.BucketManager(mac, config); + + const bucket = srcBucket; + const key = 'qiniu.mp4'; + bucketManager.stat(bucket, key, function ( + err, + respBody, + respInfo + ) { + should.not.exist(err); + respInfo.statusCode.should.be.eql(403, JSON.stringify(respInfo)); + done(); + }); + }); + }); + + describe('test bucket image source', function () { + it('test set image', function (done) { + bucketManager.image( + srcBucket, + 'http://devtools.qiniu.com/', + 'devtools.qiniu.com', + function (err, respBody, respInfo) { + should.not.exist(err, JSON.stringify(respInfo)); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + done(); + } + ); + }); + + it('test unset image', function (done) { + bucketManager.unimage( + srcBucket, + function (err, respBody, respInfo) { + should.not.exist(err, JSON.stringify(respInfo)); + respInfo.statusCode.should.be.eql(200, JSON.stringify(respInfo)); + done(); + } + ); + }); + }); + + describe('test PutPolicy', function () { + it('test build-in options (backward compatibility)', function () { + const buildInProps = { + scope: 'mocked-bucket:some/key', + isPrefixalScope: 1, + insertOnly: 1, + saveKey: 'some/key/specified.mp4', + forceSaveKey: true, + endUser: 'some-user-id', + returnUrl: 'https://mocked.qiniu.com/put-policy/return-url', + returnBody: '{"msg": "mocked"}', + callbackUrl: 'https://mocked.qiniu.com/put-policy/callback-url', + callbackHost: 'mocked.qiniu.com', + callbackBody: '{"msg": "mocked"}', + callbackBodyType: 'application/json', + callbackFetchKey: 1, + persistentOps: 'avthumb/flv|saveas/bW9ja2VkLWJ1Y2tldDpzb21lL2tleS9zcGVjaWZpZWQuZmx2Cg==', + persistentNotifyUrl: 'https://mocked.qiniu.com/put-policy/persistent-notify-url', + persistentPipeline: 'mocked-pipe', + fsizeLimit: 104857600, + fsizeMin: 10485760, + detectMime: 1, + mimeLimit: 'video/*', + deleteAfterDays: 365, + fileType: 1 + }; + const policy = new qiniu.rs.PutPolicy(buildInProps); + for (const k of Object.keys(buildInProps)) { + should.equal(policy[k], buildInProps[k], `key ${k}, ${policy[k]} not eql ${buildInProps[k]}`); + } + const flags = policy.getFlags(); + for (const k of Object.keys(buildInProps)) { + should.equal(flags[k], buildInProps[k], `key ${k}, ${policy[k]} not eql ${buildInProps[k]}`); + } + }); + + it('test expires option default value', function () { + const putPolicyOptions = { + scope: 'mocked-bucket:some/key' + }; + const policy = new qiniu.rs.PutPolicy(putPolicyOptions); + + // deviation should be less than 1sec + const deviation = policy.getFlags().deadline - Math.floor(Date.now() / 1000) - 3600; + Math.abs(deviation).should.lessThan(1); + }); + + it('test expires option', function () { + const expires = 604800; + const putPolicyOptions = { + scope: 'mocked-bucket:some/key', + expires: expires + }; + const policy = new qiniu.rs.PutPolicy(putPolicyOptions); + + // deviation should be less than 1sec + const deviation = policy.getFlags().deadline - Math.floor(Date.now() / 1000) - expires; + Math.abs(deviation).should.lessThan(1); + }); + + it('test custom options', function () { + const putPolicyOptions = { + scope: 'mocked-bucket:some/key', + mockedProp: 'mockedProp', + transform: 'some', + transform_fallback_mode: 'bar', + transform_fallback_key: 'foo' + }; + const policy = new qiniu.rs.PutPolicy(putPolicyOptions); + const flags = policy.getFlags(); + + for (const k of Object.keys(putPolicyOptions)) { + should.equal(flags[k], putPolicyOptions[k], `key ${k}, ${policy[k]} not eql ${putPolicyOptions[k]}`); + } + }); }); - }); }); diff --git a/test/rtc.test.js b/test/rtc.test.js new file mode 100644 index 00000000..bf93ce3c --- /dev/null +++ b/test/rtc.test.js @@ -0,0 +1,87 @@ +const should = require('should'); +const assert = require('assert'); +const qiniu = require('../index.js'); +const proc = require('process'); +const console = require('console'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + done(); +}); + +// eslint-disable-next-line no-undef +describe('test rtc credentials', function () { + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + + var credentials = new qiniu.Credentials(accessKey, secretKey); + var appId = null; + var appData = { + hub: 'hailong', + title: 'testtitle', + maxUsers: 10, + noAutoKickUser: true + }; + + after(function (done) { + qiniu.app.deleteApp(appId, credentials, function () { + done(); + }); + }); + + // eslint-disable-next-line no-undef + describe('test create app', function () { + // eslint-disable-next-line no-undef + it('create app', function (done) { + qiniu.app.createApp(appData, credentials, function (err, res) { + should.not.exist(err); + should.exist(res.appId); + assert.strictEqual(res.title, 'testtitle'); + appId = res.appId; + done(); + }); + }); + }); + + // eslint-disable-next-line no-undef + describe('test update app', function () { + // eslint-disable-next-line no-undef + it('update app', function (done) { + appData.title = 'testtitle2'; + qiniu.app.updateApp(appId, appData, credentials, function (err, res) { + should.not.exist(err); + assert.strictEqual(res.title, 'testtitle2'); + assert.strictEqual(res.appId, appId); + done(); + }); + }); + }); + + // eslint-disable-next-line no-undef + describe('test get app', function () { + // eslint-disable-next-line no-undef + it('get app', function (done) { + qiniu.app.getApp(appId, credentials, function (err, res) { + should.not.exist(err); + assert.strictEqual(res.title, 'testtitle2'); + assert.strictEqual(res.appId, appId); + done(); + }); + }); + }); + + // eslint-disable-next-line no-undef + describe('test delete app', function () { + // eslint-disable-next-line no-undef + it('delete app', function (done) { + qiniu.app.deleteApp(appId, credentials, function (err) { + should.not.exist(err); + done(); + }); + }); + }); +}); diff --git a/test/up_internal.test.js b/test/up_internal.test.js new file mode 100644 index 00000000..1bfd4708 --- /dev/null +++ b/test/up_internal.test.js @@ -0,0 +1,383 @@ +const should = require('should'); + +const fs = require('fs'); +const path = require('path'); + +const qiniu = require('../index'); +const { + Endpoint, + Region, + StaticRegionsProvider, + SERVICE_NAME +} = qiniu.httpc; +const { + Config +} = qiniu.conf; +const { + Zone_z1 +} = qiniu.zone; + +const { + prepareRegionsProvider, + doWorkWithRetry, + ChangeEndpointRetryPolicy, + ChangeRegionRetryPolicy, + TokenExpiredRetryPolicy +} = require('../qiniu/storage/internal'); + +describe('test upload internal module', function () { + describe('test TokenExpiredRetryPolicy', function () { + const resumeRecordFilePath = path.join(process.cwd(), 'fake-progress-record'); + + beforeEach(function () { + const fd = fs.openSync(resumeRecordFilePath, 'w'); + fs.closeSync(fd); + }); + + afterEach(function () { + try { + fs.unlinkSync(resumeRecordFilePath); + } catch (e) { + // ignore + } + }); + + it('test TokenExpiredRetryPolicy should not retry', function () { + const tokenExpiredRetryPolicy = new TokenExpiredRetryPolicy(); + const context = { + uploadApiVersion: 'v1', + resumeRecordFilePath + }; + + const mockRet = { + data: null, + resp: { + statusCode: 200 + } + }; + + // create fake progress file + return tokenExpiredRetryPolicy.initContext(context) + .then(() => { + return tokenExpiredRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(!readyToRetry); + }); + }); + + it('test TokenExpiredRetryPolicy should not by maxRetriedTimes', function () { + const tokenExpiredRetryPolicy = new TokenExpiredRetryPolicy({ + maxRetryTimes: 2 + }); + const context = { + uploadApiVersion: 'v1', + resumeRecordFilePath + }; + + const mockRet = { + data: null, + resp: { + statusCode: 701 + } + }; + + // create fake progress file + return tokenExpiredRetryPolicy.initContext(context) + .then(() => { + return tokenExpiredRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + const fd = fs.openSync(resumeRecordFilePath, 'w'); + fs.closeSync(fd); + return tokenExpiredRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + const fd = fs.openSync(resumeRecordFilePath, 'w'); + fs.closeSync(fd); + return tokenExpiredRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(!readyToRetry); + }); + }); + + it('test TokenExpiredRetryPolicy should retry v1', function () { + const tokenExpiredRetryPolicy = new TokenExpiredRetryPolicy(); + const context = { + uploadApiVersion: 'v1', + resumeRecordFilePath + }; + + const mockRet = { + data: null, + resp: { + statusCode: 701 + } + }; + + // create fake progress file + return tokenExpiredRetryPolicy.initContext(context) + .then(() => { + return tokenExpiredRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + should.ok(!fs.existsSync(resumeRecordFilePath)); + }); + }); + + it('test TokenExpiredRetryPolicy should retry v2', function () { + const tokenExpiredRetryPolicy = new TokenExpiredRetryPolicy(); + const context = { + uploadApiVersion: 'v2', + resumeRecordFilePath + }; + + const mockRet = { + data: null, + resp: { + statusCode: 612 + } + }; + + // create fake progress file + return tokenExpiredRetryPolicy.initContext(context) + .then(() => { + return tokenExpiredRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + should.ok(!fs.existsSync(resumeRecordFilePath)); + }); + }); + }); + + describe('test ChangeEndpointRetryPolicy', function () { + it('test ChangeEndpointRetryPolicy retry', function () { + const changeEndpointRetryPolicy = new ChangeEndpointRetryPolicy(); + const context = { + endpoint: new Endpoint('a'), + alternativeEndpoints: [ + new Endpoint('b'), + new Endpoint('c') + ] + }; + + const mockRet = {}; + + return changeEndpointRetryPolicy.initContext(context) + .then(() => { + return changeEndpointRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + should.equal( + context.endpoint.getValue(), + 'https://b' + ); + should.equal(context.alternativeEndpoints.length, 1); + return changeEndpointRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + should.equal( + context.endpoint.getValue(), + 'https://c' + ); + should.equal(context.alternativeEndpoints.length, 0); + return changeEndpointRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(!readyToRetry); + }); + }); + }); + + describe('test ChangeRegionRetryPolicy', function () { + const resumeRecordFilePath = path.join(process.cwd(), 'fake-progress-record'); + + it('test ChangeRegionRetryPolicy', function () { + const fd = fs.openSync(resumeRecordFilePath, 'w'); + fs.closeSync(fd); + + const changeRegionRetryPolicy = new ChangeRegionRetryPolicy(); + const context = { + resumeRecordFilePath, + serviceName: SERVICE_NAME.UP, + region: Region.fromRegionId('z0'), + alternativeRegions: [ + Region.fromRegionId('z1') + ] + }; + + const mockRet = {}; + + return changeRegionRetryPolicy.initContext(context) + .then(() => { + return changeRegionRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(readyToRetry); + should.ok(!fs.existsSync(resumeRecordFilePath)); + should.equal(context.region.regionId, 'z1'); + should.equal(context.alternativeRegions.length, 0); + should.ok(context.endpoint.getValue().includes('z1')); + should.ok(context.alternativeEndpoints.length > 0); + return changeRegionRetryPolicy.prepareRetry(context, mockRet); + }) + .then(readyToRetry => { + should.ok(!readyToRetry); + }); + }); + }); + describe('test retry', function () { + describe('test prepareRegionsProvider', function () { + it('test prepareRegionsProvider with config provider', function () { + const staticRegionsProvider = new StaticRegionsProvider([ + Region.fromRegionId('z1') + ]); + const config = new Config({ + regionsProvider: staticRegionsProvider + }); + + return prepareRegionsProvider({ + config, + bucketName: 'mock-bucket', + accessKey: 'mock-ak' + }) + .then(regionsProvider => { + should.equal(regionsProvider, staticRegionsProvider); + }); + }); + + it('test prepareRegionsProvider with config zone', function () { + const config = new Config({ + zone: Zone_z1 + }); + + return prepareRegionsProvider({ + config, + bucketName: 'mock-bucket', + accessKey: 'mock-ak' + }) + .then(regionsProvider => { + should.ok(regionsProvider instanceof StaticRegionsProvider); + return regionsProvider.getRegions(); + }) + .then(regions => { + should.equal(regions.length, 1); + const [r] = regions; + should.not.exist(r.regionId); + const actualServiceHost = Object.keys(r.services) + .reduce((services, serviceKey) => { + services[serviceKey] = r.services[serviceKey].map(e => e.host); + return services; + }, {}); + const expectServiceHost = { + [SERVICE_NAME.UP]: Zone_z1.srcUpHosts.concat(Zone_z1.cdnUpHosts), + [SERVICE_NAME.IO]: [Zone_z1.ioHost], + [SERVICE_NAME.RS]: [Zone_z1.rsHost], + [SERVICE_NAME.RSF]: [Zone_z1.rsfHost], + [SERVICE_NAME.API]: [Zone_z1.apiHost], + [SERVICE_NAME.UC]: [], + [SERVICE_NAME.S3]: [] + }; + should.deepEqual(actualServiceHost, expectServiceHost); + }); + }); + }); + + describe('test retry regions', function () { + function getTestData (options) { + options = options || {}; + const failedTimes = options.failedTimes || 0; + const triedEndpoint = []; + function workWithEndpoint (endpoint) { + triedEndpoint.push(endpoint); + return Promise.resolve({ + err: null, + ret: { + msg: 'ok' + }, + info: { + statusCode: triedEndpoint.length <= failedTimes ? -1 : 200 + } + }); + } + + const staticRegionsProvider = new StaticRegionsProvider([ + Region.fromRegionId('z1'), + Region.fromRegionId('z2') + ]); + + return { + triedEndpoint, + workWithEndpoint, + staticRegionsProvider + }; + } + + it('test retry regions with ok and no change region', function () { + const { + staticRegionsProvider, + triedEndpoint, + workWithEndpoint + } = getTestData({ + failedTimes: 0 + }); + + return doWorkWithRetry({ + workFn: workWithEndpoint, + regionsProvider: staticRegionsProvider, + retryPolicies: [ + new ChangeEndpointRetryPolicy(), + new ChangeRegionRetryPolicy() + ] + }) + .then(({ data, resp }) => { + should.equal(resp.statusCode, 200); + should.equal(data.msg, 'ok'); + should.equal(triedEndpoint.length, 1); + const [endpoint] = triedEndpoint; + should.ok(endpoint.getValue().includes('z1')); + }); + }); + + it('test retry regions with change region', function () { + const { + staticRegionsProvider, + triedEndpoint, + workWithEndpoint + } = getTestData({ + failedTimes: 3 + }); + + const resumeRecordFilePath = path.join( + process.cwd(), + 'progress-record-file' + ); + const fd = fs.openSync(resumeRecordFilePath, 'w'); + fs.closeSync(fd); + + return doWorkWithRetry({ + workFn: workWithEndpoint, + resumeRecordFilePath: resumeRecordFilePath, + regionsProvider: staticRegionsProvider, + retryPolicies: [ + new ChangeEndpointRetryPolicy(), + new ChangeRegionRetryPolicy() + ] + }) + .then(({ data, resp }) => { + should.equal(resp.statusCode, 200); + should.equal(data.msg, 'ok'); + should.ok(!fs.existsSync(resumeRecordFilePath)); + should.equal(triedEndpoint.length, 4); + }); + }); + }); + }); +}); diff --git a/test/util.test.js b/test/util.test.js new file mode 100644 index 00000000..75a3d22c --- /dev/null +++ b/test/util.test.js @@ -0,0 +1,491 @@ +const should = require('should'); +const qiniu = require('../index.js'); +const proc = require('process'); +const console = require('console'); + +// eslint-disable-next-line no-undef +before(function (done) { + if (!process.env.QINIU_ACCESS_KEY || !process.env.QINIU_SECRET_KEY || !process.env.QINIU_TEST_BUCKET || !process.env.QINIU_TEST_DOMAIN) { + console.log('should run command `source test-env.sh` first\n'); + process.exit(0); + } + done(); +}); + +describe('test util functions', function () { + var accessKey = proc.env.QINIU_ACCESS_KEY; + var secretKey = proc.env.QINIU_SECRET_KEY; + var bucket = proc.env.QINIU_TEST_BUCKET; + var mac = new qiniu.auth.digest.Mac(accessKey, secretKey); + var config = new qiniu.conf.Config(); + var bucketManager = new qiniu.rs.BucketManager(mac, config); + + describe('test prepareZone', function () { + it('test prepareZone', function (done) { + config.zone = qiniu.zone.Zone_z0; + qiniu.util.prepareZone(bucketManager, bucketManager.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.equal(bucketManager, ctx); + done(); + }); + }); + + it('test prepareZone error', function (done) { + config.zone = null; + qiniu.util.prepareZone(bucketManager, 'no_ak', 'no_bucket', function (err, ctx) { + should.exist(err); + done(); + }); + }); + + it('test prepareZone with null', function (done) { + config.zone = null; + qiniu.util.prepareZone(bucketManager, bucketManager.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.equal(bucketManager, ctx); + done(); + }); + }); + + it('test prepareZone with zone expired', function (done) { + config.zoneExpire = -1; + qiniu.util.prepareZone(bucketManager, bucketManager.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.equal(bucketManager, ctx); + done(); + }); + }); + + it('test prepareZone with zone expired', function (done) { + config.zoneExpire = parseInt(Date.now() / 1000) - 10; + qiniu.util.prepareZone(bucketManager, bucketManager.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.equal(bucketManager, ctx); + done(); + }); + }); + + it('test prepareZone with zone not expired', function (done) { + config.zoneExpire = parseInt(Date.now() / 1000) + 10; + qiniu.util.prepareZone(bucketManager, bucketManager.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.equal(bucketManager, ctx); + done(); + }); + }); + + it('test formatDateUTC', function () { + const caseList = [ + { + date: new Date('2022-05-19T03:28:46.816Z'), + layout: 'YYYY-MM-DD HH:mm:ss.SSS', + expect: '2022-05-19 03:28:46.816' + }, + { + date: new Date('2022-05-19T03:28:46.816Z'), + layout: 'YYYYMMDDTHHmmssZ', + expect: '20220519T032846Z' + } + ]; + + for (let i = 0; i < caseList.length; i++) { + const actual = qiniu.util.formatDateUTC(caseList[i].date, caseList[i].layout); + const expect = caseList[i].expect; + should.equal(actual, expect); + } + }); + + it('test canonicalMimeHeaderKey', function () { + const fieldNames = [ + ':status', + ':x-test-1', + ':x-Test-2', + 'content-type', + 'CONTENT-LENGTH', + 'oRiGin', + 'ReFer', + 'Last-Modified', + 'acCePt-ChArsEt', + 'x-test-3', + 'cache-control', + '七牛' + ]; + const expectCanonicalFieldNames = [ + ':status', + ':x-test-1', + ':x-Test-2', + 'Content-Type', + 'Content-Length', + 'Origin', + 'Refer', + 'Last-Modified', + 'Accept-Charset', + 'X-Test-3', + 'Cache-Control', + '七牛' + ]; + should.equal(fieldNames.length, expectCanonicalFieldNames.length); + for (let i = 0; i < fieldNames.length; i++) { + should.equal(qiniu.util.canonicalMimeHeaderKey(fieldNames[i]), expectCanonicalFieldNames[i]); + } + }); + + it('test generateAccessTokenV2', function () { + const mac = new qiniu.auth.digest.Mac('ak', 'sk'); + + const testCases = [ + { + method: 'GET', + host: undefined, + url: 'http://rs.qbox.me', + qheaders: { + 'x-Qiniu-': 'a', + 'X-qIniu': 'b', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + contentType: 'application/x-www-form-urlencoded', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:CK4wBVOL6sLbVE4G4mrXqL_yEc4=' + }, + { + method: 'GET', + host: undefined, + url: 'http://rs.qbox.me', + qheaders: { + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + contentType: 'application/x-www-form-urlencoded', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:CK4wBVOL6sLbVE4G4mrXqL_yEc4=' + }, + { + method: 'GET', + host: undefined, + url: 'http://rs.qbox.me', + qheaders: { + 'Content-Type': 'application/json' + }, + contentType: 'application/json', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:ksh7bJBnBzFO0yxJ_tLLUcg0csM=' + }, + { + method: 'POST', + host: undefined, + url: 'http://rs.qbox.me', + qheaders: { + 'Content-Type': 'application/json', + 'X-Qiniu': 'b' + }, + contentType: 'application/json', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:IlW01tHjGQ0pGPXV_3jjR1AdD34=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com', + qheaders: { + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b', + 'Content-Type': 'application/x-www-form-urlencoded' + }, + contentType: 'application/x-www-form-urlencoded', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:156x8Q4x1zadPcAyMRVDsioIyAk=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com', + qheaders: { + 'Content-Type': 'application/json', + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: 'application/json', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:eOaX4RziJPW9ywnJ02jshmEMfhI=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com', + qheaders: { + 'Content-Type': 'application/octet-stream', + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: 'application/octet-stream', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:GQQrYvDCdN_RaVjyJC7hIkv5TYk=' + }, + { + method: 'gET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com', + qheaders: { + 'Content-Type': 'application/json', + 'X-Qiniu-Bbb': 'BBB', + 'x-qIniu-aAa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: 'application/json', + body: '{"name": "test"}', + exceptSignToken: 'Qiniu ak:eOaX4RziJPW9ywnJ02jshmEMfhI=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com', + qheaders: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: 'application/x-www-form-urlencoded', + body: 'name=test&language=go', + exceptSignToken: 'Qiniu ak:A5PMXECSPZQxitJqLj0op2B2GEM=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com', + qheaders: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD' + }, + contentType: 'application/x-www-form-urlencoded', + body: 'name=test&language=go', + exceptSignToken: 'Qiniu ak:A5PMXECSPZQxitJqLj0op2B2GEM=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com/mkfile/sdf.jpg', + qheaders: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: 'application/x-www-form-urlencoded', + body: 'name=test&language=go', + exceptSignToken: 'Qiniu ak:fkRck5_LeyfwdkyyLk-hyNwGKac=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com/mkfile/sdf.jpg?s=er3&df', + qheaders: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: 'application/x-www-form-urlencoded', + body: 'name=test&language=go', + exceptSignToken: 'Qiniu ak:PUFPWsEUIpk_dzUvvxTTmwhp3p4=' + }, + { + method: 'GET', + host: 'upload.qiniup.com', + url: 'http://upload.qiniup.com/mkfile/sdf.jpg?s=er3&df', + qheaders: { + 'X-Qiniu-Bbb': 'BBB', + 'X-Qiniu-Aaa': 'DDD', + 'X-Qiniu-': 'a', + 'X-Qiniu': 'b' + }, + contentType: undefined, + body: 'name=test&language=go', + exceptSignToken: 'Qiniu ak:PUFPWsEUIpk_dzUvvxTTmwhp3p4=' + } + ]; + for (const testCase of testCases) { + should.equal( + qiniu.util.generateAccessTokenV2( + mac, + testCase.url, + testCase.method, + testCase.contentType, + testCase.body, + testCase.qheaders + ), + testCase.exceptSignToken + ); + } + }); + + it('test encodedEntry', function () { + const caseList = [ + { + msg: 'normal', + bucket: 'qiniuphotos', + key: 'gogopher.jpg', + expect: 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + msg: 'key empty', + bucket: 'qiniuphotos', + key: '', + expect: 'cWluaXVwaG90b3M6' + }, + { + msg: 'key undefined', + bucket: 'qiniuphotos', + key: undefined, + expect: 'cWluaXVwaG90b3M=' + }, + { + msg: 'key need replace plus symbol', + bucket: 'qiniuphotos', + key: '012ts>a', + expect: 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + msg: 'key need replace slash symbol', + bucket: 'qiniuphotos', + key: '012ts?a', + expect: 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ]; + + for (let i = 0; i < caseList.length; i++) { + const actual = qiniu.util.encodedEntry(caseList[i].bucket, caseList[i].key); + const expect = caseList[i].expect; + const msg = caseList[i].msg; + should.equal(actual, expect, msg); + } + }); + + it('test decodedEntry', function () { + const caseList = [ + { + msg: 'normal', + expect: { + bucket: 'qiniuphotos', + key: 'gogopher.jpg' + }, + entry: 'cWluaXVwaG90b3M6Z29nb3BoZXIuanBn' + }, + { + msg: 'key empty', + expect: { + bucket: 'qiniuphotos', + key: '' + }, + entry: 'cWluaXVwaG90b3M6' + }, + { + msg: 'key undefined', + expect: { + bucket: 'qiniuphotos', + key: undefined + }, + entry: 'cWluaXVwaG90b3M=' + }, + { + msg: 'key need replace plus symbol', + expect: { + bucket: 'qiniuphotos', + key: '012ts>a' + }, + entry: 'cWluaXVwaG90b3M6MDEydHM-YQ==' + }, + { + msg: 'key need replace slash symbol', + expect: { + bucket: 'qiniuphotos', + key: '012ts?a' + }, + entry: 'cWluaXVwaG90b3M6MDEydHM_YQ==' + } + ]; + + for (let i = 0; i < caseList.length; i++) { + const [actualBucket, actualKey] = qiniu.util.decodedEntry(caseList[i].entry); + const expect = caseList[i].expect; + const msg = caseList[i].msg; + should.deepEqual({ + bucket: actualBucket, + key: actualKey + }, expect, msg); + } + }); + }); + + describe('test prepareZone with change hosts config', function () { + let bucketManagerNoCtxCache = new qiniu.rs.BucketManager(mac, config); + + beforeEach(function () { + const noCacheConfig = new qiniu.conf.Config(); + bucketManagerNoCtxCache = new qiniu.rs.BucketManager(mac, noCacheConfig); + }); + + afterEach(function () { + qiniu.conf.UC_HOST = 'uc.qbox.me'; + qiniu.conf.QUERY_REGION_HOST = 'kodo-config.qiniuapi.com'; + qiniu.conf.QUERY_REGION_BACKUP_HOSTS = [ + 'uc.qbox.me', + 'api.qiniu.com' + ]; + }); + + it('test prepareZone with custom query domain', function (done) { + should.not.exist(bucketManagerNoCtxCache.config.zone); + + qiniu.conf.QUERY_REGION_HOST = 'uc.qbox.me'; + + qiniu.util.prepareZone(bucketManagerNoCtxCache, bucketManagerNoCtxCache.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.exist(ctx.config.zone); + done(); + }); + }); + + it('test prepareZone with backup domain', function (done) { + should.not.exist(bucketManagerNoCtxCache.config.zone); + + qiniu.conf.QUERY_REGION_HOST = 'fake-uc.csharp.qiniu.com'; + qiniu.conf.QUERY_REGION_BACKUP_HOSTS = [ + 'unavailable-uc.csharp.qiniu.com', + 'uc.qbox.me' + ]; + + qiniu.util.prepareZone(bucketManagerNoCtxCache, bucketManagerNoCtxCache.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.exist(ctx.config.zone); + done(); + }); + }); + + it('test prepareZone with uc and backup domains', function (done) { + should.not.exist(bucketManagerNoCtxCache.config.zone); + + qiniu.conf.UC_HOST = 'fake-uc.csharp.qiniu.com'; + qiniu.conf.QUERY_REGION_BACKUP_HOSTS = [ + 'unavailable-uc.csharp.qiniu.com', + 'uc.qbox.me' + ]; + + qiniu.util.prepareZone(bucketManagerNoCtxCache, bucketManagerNoCtxCache.mac.accessKey, bucket, function (err, ctx) { + should.not.exist(err); + should.exist(ctx.config.zone); + done(); + }); + }); + }); +});