Merge branch 'master' into add-dburi-option

telefonicaid · Aug 24, 2023 · 8d0617c · 8d0617c
2 parents faddd42 + 9b520ae
commit 8d0617c
Show file tree

Hide file tree

Showing 24 changed files with 1,060 additions and 83 deletions.
diff --git a/.github/workflows/ciimage.yml b/.github/workflows/ciimage.yml
diff --git a/.github/workflows/functional.yml b/.github/workflows/functional.yml
@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   functional:

diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml
@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  TEST_IMAGE_NAME: fiware/orion-ci:deb
+  TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   unit:

diff --git a/.github/workflows/valgrind-nocache.yml b/.github/workflows/valgrind-nocache.yml
@@ -17,7 +17,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   valgrind:

diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml
@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-   TEST_IMAGE_NAME: fiware/orion-ci:deb
+   TEST_IMAGE_NAME: telefonicaiot/fiware-orion:ci
 
 jobs:
   valgrind:

diff --git a/CHANGES_NEXT_RELEASE b/CHANGES_NEXT_RELEASE
@@ -1,2 +1,4 @@
+- Add: servicePath field to builtin attributes (#2877)
 - Fix: logDeprecate not working correctly (`geo:json` wrongly considered as deprecated)
+- Fix: improve error traces (#4387)
 - Add: CLI parameter -dbUri / env var ORION_MONGO_URI (#3794)
diff --git a/ci/README.jp.md b/ci/README.jp.md
@@ -1,13 +1,13 @@
 ## 概要
 このリポジトリでは GitHub Actions が有効になっているため、マージを許可する前に各プルリクエストがチェックされます。
-このシステムは、新しい PR がマスターに到達するたびにマスターブランチから構築される `fiware/orion-ci:deb` に基づいており、
+このシステムは、新しい PR がマスターに到達するたびにマスターブランチから構築される `telefonicaiot/fiware-orion:ci` に基づいており、
 すべてのビルド依存関係がオンボードにあるクリーンな環境を提供します。この Docker のビルドに使用される Dockerfile は、
 `ci/deb` ディレクトリにあります。
 
-テスト対象の PR ブランチが変更されたため、`fiware/orion-ci:deb` は再構築されないことに注意してください。したがって、
+テスト対象の PR ブランチが変更されたため、`telefonicaiot/fiware-orion:ci` は再構築されないことに注意してください。したがって、
 新しいライブラリまたはベースシステムを必要とする機能を開発している場合は、そのようなライブラリまたはベースシステムを
 `ci/deb/build-dep.sh` および/または `Dockerfile` に追加する PR を*最初に*実行する必要があります。その PR がマスターに
-マージされ、`fiware/orion-ci:deb` が再構築されると (Docker Hub の https://hub.docker.com/r/fiware/orion-ci/builds
+マージされ、`telefonicaiot/fiware-orion:ci` が再構築されると (Docker Hub の https://hub.docker.com/r/telefonicaiot/fiware-orion/builds
 で進行状況を確認)、新しい機能は、GitHub アクションでテストする準備ができています。
 
 GitHub Actions チェックは段階に分かれており、 "サポートされているテスト" セクションで説明されています。
@@ -31,23 +31,23 @@ CI の現在のバージョンは以下をサポートします:
 イメージをダウンロードするには:
 
 ```
-docker pull fiware/orion-ci:deb
+docker pull telefonicaiot/fiware-orion:ci
 ```
 
 たとえば、GitHub Actions と同じ方法でイメージを実行するには、次のようにします:
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb build -miqts functional
+docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci build -miqts functional
 ```
 
 インタラクティブな bash を使用してイメージを実行するには:
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host -ti -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb bash
+docker run --network host -ti -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci bash
 ```
 
 bash シェルを起動したら、同様の実行を行うことができます:

diff --git a/ci/README.md b/ci/README.md
@@ -1,13 +1,13 @@
 ## Overview
 GitHub Actions is enabled in this repository so each pull request is checked before being allowed to merge.
-The system is based on `fiware/orion-ci:deb` which is built from master branch each time a new PR lands in master,
+The system is based on `telefonicaiot/fiware-orion:ci` which is built from master branch each time a new PR lands in master,
 providing a clean environment with all build dependencies onboard. The Dockerfile used to build this docker is available
 in the `ci/deb` directory.
 
-Note that `fiware/orion-ci:deb` is *not* rebuilt due to changes in the PR branch under test. Thus, if you are developing
+Note that `telefonicaiot/fiware-orion:ci` is *not* rebuilt due to changes in the PR branch under test. Thus, if you are developing
 a functionality that requires a new library or base system you need to do *first* a PR adding such library or base system
-to `ci/deb/build-dep.sh` and/or `Dockerfile`. Once that PR gets merged into master and `fiware/orion-ci:deb` gets rebuild
-(checking progress in Dockerhub at: https://hub.docker.com/r/fiware/orion-ci/builds) your PR branch with the new 
+to `ci/deb/build-dep.sh` and/or `Dockerfile`. Once that PR gets merged into master and `telefonicaiot/fiware-orion:ci` gets rebuild
+(checking progress in Dockerhub at: https://hub.docker.com/r/telefonicaiot/fiware-orion/builds) your PR branch with the new 
 functionality is ready to be tested with GitHub Actions.
 
 The GitHub Actions checks are divided into stages, which are described in "Supported tests" section.
@@ -30,23 +30,23 @@ the following cheatsheet can be useful:
 To download the image:
 
 ```
-docker pull fiware/orion-ci:deb
+docker pull telefonicaiot/fiware-orion:ci
 ```
 
 To run the image in the same way that GitHub Actions does, for instance:
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb build -miqts functional
+docker run --network host --rm -e CB_NO_CACHE=ON -e FT_FROM_IX=1201 -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci build -miqts functional
 ```
 
 To run the image using an interactive bash on it
 
 ```
 # Check that MongoDB server is running in your localhost:27017
 cd /path/to/fiware-orion
-docker run --network host -ti -v $(pwd):/opt/fiware-orion fiware/orion-ci:deb bash
+docker run --network host -ti -v $(pwd):/opt/fiware-orion telefonicaiot/fiware-orion:ci bash
 ```
 
 Once have a bash shell, you can do the same execution:

diff --git a/doc/manuals.jp/orion-api.md b/doc/manuals.jp/orion-api.md
@@ -643,9 +643,9 @@ Orion は階層スコープをサポートしているため、エンティテ
     -   `entityUpdate`: 通知をトリガーする更新が更新であったが、実際の変更ではなかった場合
     -   `entityChange`: 通知をトリガーする更新が実際の変更を伴う更新であった場合、または実際の変更ではなく
         `forcedUpdate` が使用された場合
-    -   `entityDelete`: 通知をトリガーする更新がエンティティの削除操作であった場合。この場合、
-        [`condition`](#subscriptionsubjectcondition) 内の `attrs` フィールドは無視されます (エンティティを削除する通常
-        の方法、たとえば `DELETE /v2/entities/E` には属性が含まれないことに注意してください)
+    -   `entityDelete`: 通知をトリガーする更新がエンティティの削除操作であった場合
+
+-   `servicePath` (タイプ: `Text`): エンティティが属する[サービス・パス](#service-path)を指定します。
 
 通常の属性と同様に、`q` フィルタと `orderBy` (`alterationType` を除く) で使用できます。
 ただし、リソース URLs では使用できません。

diff --git a/doc/manuals/orion-api.md b/doc/manuals/orion-api.md
@@ -674,6 +674,8 @@ the subscriptions based in alteration type features (see [Subscription based in
    * `entityChange` if the update that triggers the notification was an update with an actual change or not an actual change but with `forcedUpdate` in use
    * `entityDelete` if the update that triggers the notification was a entity delete operation
 
+* `servicePath` (type: `Text`): specifies the [service path](#service-path) to which the entity belongs.
+
 Like regular attributes, they can be used in `q` filters and in `orderBy` (except `alterationType`).
 However, they cannot be used in resource URLs.
 

diff --git a/doc/requirements.txt b/doc/requirements.txt
@@ -1,4 +1,4 @@
 mkdocs==1.2.3
-Pygments==2.9.0
+Pygments==2.15.0
 Markdown==3.3.4
 jinja2==3.0.0
diff --git a/doc/roadmap.md b/doc/roadmap.md
@@ -17,7 +17,7 @@ any time.
 
 Disclaimer:
 
-* This section has been last updated in March 2022. Please take into account its 
+* This section has been last updated in August 2023. Please take into account its 
   content could be obsolete.
 * Note we develop this software in Agile way, so development plan is continuously 
   under review. Thus, this roadmap has to be understood as rough plan of features 
@@ -32,12 +32,12 @@ Disclaimer:
 The following list of features are planned to be addressed in the short term,
 and incorporated into the coming release(s) of the product:
 
+- MQTT Retain flag [#4388](https://github.com/telefonicaid/fiware-orion/issues/4388)
 - Allow multiple types in entity to support UNE 178503 requirements ([#3638](https://github.com/telefonicaid/fiware-orion/issues/3638))
-- ~New subscripition modes (create only, update only, delete only and combinations)~ ([#1494](https://github.com/telefonicaid/fiware-orion/issues/1494)
 - Pattern/filter batch updates ([#2389](https://github.com/telefonicaid/fiware-orion/issues/2389))
 - Notification endpoint alias ([#3655](https://github.com/telefonicaid/fiware-orion/issues/3655))
 - Aggregation operations API ([#3816](https://github.com/telefonicaid/fiware-orion/issues/3816))
-- ~Custom notifications: simplifying sending JSON requests~ ([#2560](https://github.com/telefonicaid/fiware-orion/issues/2560))
+- Rework commands (and deprecate registration API) ([#4397](https://github.com/telefonicaid/fiware-orion/issues/4397))
 
 
 ## Medium term
@@ -46,16 +46,20 @@ The following list of features are planned to be addressed in the medium term,
 typically within the subsequent release(s) generated in the next **9 months**
 after next planned release:
 
-
-- Advanced query language
-- Signed entities
+- Advanced query language ([#4395](https://github.com/telefonicaid/fiware-orion/issues/4395))
+- Signed entities ([#4398](https://github.com/telefonicaid/fiware-orion/issues/4398))
 - Dynamic / high order attribute values (e.g. an attribute being a sum of two other attributes)
  supported by a Expressions Language - help wanted 
 ([#4004](https://github.com/telefonicaid/fiware-orion/issues/4004)),
 ([#3815](https://github.com/telefonicaid/fiware-orion/issues/3815))
-- Rework commands
 - Service provisioning API (pools, etc.) 
 (based in [#3843](https://github.com/telefonicaid/fiware-orion/issues/3843))
+- Advanced subscription management
+   - Subscription debug mode (precise statistics consolidation, keep recent history of notifications sent, etc.) ([#4399](https://github.com/telefonicaid/fiware-orion/issues/4399))
+   - Custom ID subscription ([#4400](https://github.com/telefonicaid/fiware-orion/issues/4400))
+   - Query Subscrition by fields (endpoint, attributes...) ([#4392](https://github.com/telefonicaid/fiware-orion/issues/4392))
+   - Flag to disable metadata in subscriptions ([#4401](https://github.com/telefonicaid/fiware-orion/issues/4401))
+- Remove registration API ([#4402](https://github.com/telefonicaid/fiware-orion/pull/4402))
 
 ## Long term
 
@@ -71,6 +75,8 @@ you wish to get involved in the implementation or influence the roadmap
 
 The following list contains all features that were in the roadmap and have already been implemented.
 
+- Custom notifications: simplifying sending JSON requests ([#2560](https://github.com/telefonicaid/fiware-orion/issues/2560)) ([3.8.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.8.0))
+- New subscripition modes (create only, update only, delete only and combinations) ([#1494](https://github.com/telefonicaid/fiware-orion/issues/1494)) ([3.7.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.7.0))
 - Per sub/reg HTTP timeout ([#3842](https://github.com/telefonicaid/fiware-orion/issues/3842)) 
 ([3.3.0](https://github.com/telefonicaid/fiware-orion/releases/tag/3.3.0))
 - Attribute update operators (inc, push, etc.) ([#3814](https://github.com/telefonicaid/fiware-orion/issues/3814))

diff --git a/docker/README.jp.md b/docker/README.jp.md
@@ -43,7 +43,7 @@ Orion Context Broker を試してみたいし、データベースについて
 
 > `--nojournal` に関しては、それはプロダクション利用では推奨されていませんが、Orion コンテナが高速で、DB が見つからず準備ができていない場合に、mongo コンテナの起動を高速化し、いくつかの競合状態の問題を回避します。
 
-数秒後に、Context broker を実行し、ポート1026でリッスンする必要があります。
+数秒後、Context Broker が実行され、ポート 1026 をリッスンします。
 
 以下を実行し、動作することを確認します。
 

diff --git a/src/lib/common/globals.h b/src/lib/common/globals.h
@@ -95,6 +95,7 @@
 #define DATE_MODIFIED   "dateModified"
 #define DATE_EXPIRES    "dateExpires"
 #define ALTERATION_TYPE "alterationType"
+#define SERVICE_PATH    "servicePath"
 #define ALL_ATTRS       "*"
 
 

diff --git a/src/lib/mongoBackend/MongoCommonUpdate.cpp b/src/lib/mongoBackend/MongoCommonUpdate.cpp
@@ -316,7 +316,7 @@ static bool attrValueChanges(const orion::BSONObj& attr, ContextAttribute* caP,
     return caP->valueType != orion::ValueTypeNull;
 
   default:
-    LM_E(("Runtime Error (unknown attribute value type in DB: %d)", getFieldF(attr, ENT_ATTRS_VALUE).type()));
+    LM_E(("Runtime Error (unknown attribute value type in DB: %d on attribute %s)", getFieldF(attr, ENT_ATTRS_VALUE).type(), caP->name.c_str()));
     return false;
   }
 }
@@ -442,7 +442,7 @@ static ChangeType mergeAttrInfo
       break;
 
     default:
-      LM_E(("Runtime Error (unknown attribute value type in DB: %d)", getFieldF(attr, ENT_ATTRS_VALUE).type()));
+      LM_E(("Runtime Error (unknown attribute value type in DB: %d on attribute %s)", getFieldF(attr, ENT_ATTRS_VALUE).type(), caP->name.c_str()));
     }
   }
 

diff --git a/src/lib/mongoBackend/MongoGlobal.cpp b/src/lib/mongoBackend/MongoGlobal.cpp
@@ -838,6 +838,11 @@ static std::string sortCriteria(const std::string& sortToken)
     return ENT_MODIFICATION_DATE;
   }
 
+  if (sortToken == SERVICE_PATH)
+  {
+    return std::string("_id.") + ENT_SERVICE_PATH;
+  }
+
   if (sortToken == ENT_ENTITY_ID)
   {
     return std::string("_id.") + ENT_ENTITY_ID;
@@ -1315,6 +1320,11 @@ void addBuiltins(ContextElementResponse* cerP, const std::string& alterationType
     addIfNotPresentAttr(&cerP->entity, ALTERATION_TYPE, DEFAULT_ATTR_STRING_TYPE, alterationType);
   }
 
+  // servicePath
+  if (!cerP->entity.servicePath.empty())
+  {
+    addIfNotPresentAttr(&cerP->entity, SERVICE_PATH, DEFAULT_ATTR_STRING_TYPE, cerP->entity.servicePath);
+  }
 
   for (unsigned int ix = 0; ix < cerP->entity.attributeVector.size(); ix++)
   {

diff --git a/src/lib/ngsi/ContextAttribute.cpp b/src/lib/ngsi/ContextAttribute.cpp
@@ -133,7 +133,7 @@ void ContextAttribute::bsonAppendAttrValue
       break;
 
     default:
-      LM_E(("Runtime Error (unknown attribute type: %d)", valueType));
+      LM_E(("Runtime Error (unknown attribute value type: %d on attribute %s)", valueType, name.c_str()));
   }
 }
 
@@ -217,7 +217,7 @@ bool ContextAttribute::calculateOperator
       return false;
 
     default:
-      LM_E(("Runtime Error (unknown attribute type: %d)", valueType));
+      LM_E(("Runtime Error (unknown attribute value type: %d on attribute %s)", valueType, name.c_str()));
       return false;
     }
 
@@ -247,7 +247,7 @@ bool ContextAttribute::calculateOperator
       return false;
 
     default:
-      LM_E(("Runtime Error (unknown attribute type: %d)", valueType));
+      LM_E(("Runtime Error (unknown attribute value type: %d on attribute %s)", valueType, name.c_str()));
       return false;
     }
   }
@@ -258,7 +258,7 @@ bool ContextAttribute::calculateOperator
   }
   else
   {
-    LM_E(("Runtime Error (uknown operator: %s)", op.c_str()));
+    LM_E(("Runtime Error (unknown operator: %s)", op.c_str()));
     return false;
   }
 

diff --git a/src/lib/ngsi/ContextAttributeVector.cpp b/src/lib/ngsi/ContextAttributeVector.cpp
@@ -389,7 +389,7 @@ void ContextAttributeVector::fill
         break;
 
       default:
-        LM_E(("Runtime Error (unknown attribute value type in DB: %d)", getFieldF(attr, ENT_ATTRS_VALUE).type()));
+        LM_E(("Runtime Error (unknown attribute value type in DB: %d on attribute %s)", getFieldF(attr, ENT_ATTRS_VALUE).type(), ca.name.c_str()));
       }
     }
 
@@ -669,7 +669,7 @@ void ContextAttributeVector::applyUpdateOperators(void)
             break;
 
           default:
-            LM_E(("Runtime Error (unknown attribute type: %d)", upOp->valueType));
+            LM_E(("Runtime Error (unknown attribute value type: %d on attribute %s)", upOp->valueType, vec[ix]->name.c_str()));
           }
 
           // Replace old compound value (with $push) with the new one ([])
@@ -705,7 +705,7 @@ void ContextAttributeVector::applyUpdateOperators(void)
             break;
 
           default:
-            LM_E(("Runtime Error (unknown attribute type: %d)", upOp->valueType));
+            LM_E(("Runtime Error (unknown attribute value type: %d on attribute %s)", upOp->valueType, vec[ix]->name.c_str()));
           }
 
           // Replace old compound value (with $push) with the new one ([])
@@ -726,7 +726,7 @@ void ContextAttributeVector::applyUpdateOperators(void)
         }
         else
         {
-          LM_E(("Runtime Error (uknown operator: %s", op.c_str()));
+          LM_E(("Runtime Error (unknown operator: %s", op.c_str()));
         }
       }
     }

diff --git a/src/lib/rest/StringFilter.cpp b/src/lib/rest/StringFilter.cpp
@@ -1835,6 +1835,10 @@ bool StringFilter::mongoFilterPopulate(std::string* errorStringP)
     {
       k = ENT_MODIFICATION_DATE;
     }
+    else if (left == SERVICE_PATH)
+    {
+      k = std::string("_id.") + ENT_SERVICE_PATH;
+    }
     else if (left == itemP->attributeName + "." + ENT_ATTRS_MD "." + NGSI_MD_DATECREATED)
     {
       k = std::string(ENT_ATTRS) + "." + itemP->attributeName + "." + ENT_ATTRS_CREATION_DATE;