Skip to content

Commit

Permalink
Merge branch 'main' into jenkins-add-windows-pipeline
Browse files Browse the repository at this point in the history
  • Loading branch information
big-r81 authored Jan 20, 2023
2 parents d3de330 + 3b6ebde commit a9be938
Show file tree
Hide file tree
Showing 6 changed files with 102 additions and 12 deletions.
5 changes: 5 additions & 0 deletions Makefile.win
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,11 @@ elixir-check-formatted: elixir-init
elixir-credo: elixir-init
@mix credo

.PHONY: build-report
# target: build-report - Generate a build report
build-report:
@$(PYTHON) build-aux/show-test-results.py --suites=10 --tests=10 > test-results.log

.PHONY: check-qs
# target: check-qs - Run query server tests (ruby and rspec required!)
check-qs:
Expand Down
2 changes: 1 addition & 1 deletion rebar.config.script
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ DepDescs = [
{jiffy, "jiffy", {tag, "CouchDB-1.0.9-2"}},
{mochiweb, "mochiweb", {tag, "v3.1.1"}},
{meck, "meck", {tag, "0.9.2"}},
{recon, "recon", {tag, "2.5.2"}}
{recon, "recon", {tag, "2.5.3"}}
].

WithProper = lists:keyfind(with_proper, 1, CouchConfig) == {with_proper, true}.
Expand Down
22 changes: 15 additions & 7 deletions src/couch_replicator/src/couch_replicator_worker.erl
Original file line number Diff line number Diff line change
Expand Up @@ -297,17 +297,25 @@ queue_fetch_loop(#fetch_st{} = St) ->
{changes, ChangesManager, Changes, ReportSeq} ->
% Find missing revisions (POST to _revs_diff)
{IdRevs, RdSt1} = find_missing(Changes, Target, Parent, RdSt),
{Docs, BgSt1} = bulk_get(UseBulkGet, Source, IdRevs, Parent, BgSt),
% Documents without attachments can be uploaded right away
BatchFun = fun({_, #doc{} = Doc}) ->
ok = gen_server:call(Parent, {batch_doc, Doc}, infinity)
% Filter out and handle design docs individually
DDocFilter = fun
({<<?DESIGN_DOC_PREFIX, _/binary>>, _Rev}, _PAs) -> true;
({_Id, _Rev}, _PAs) -> false
end,
lists:foreach(BatchFun, lists:sort(maps:to_list(Docs))),
% Fetch individually if _bulk_get failed or there are attachments
DDocIdRevs = maps:filter(DDocFilter, IdRevs),
FetchFun = fun({Id, Rev}, PAs) ->
ok = gen_server:call(Parent, {fetch_doc, {Id, [Rev], PAs}}, infinity)
end,
maps:map(FetchFun, maps:without(maps:keys(Docs), IdRevs)),
maps:map(FetchFun, DDocIdRevs),
% IdRevs1 is all the docs without design docs. Bulk get those.
IdRevs1 = maps:without(maps:keys(DDocIdRevs), IdRevs),
{Docs, BgSt1} = bulk_get(UseBulkGet, Source, IdRevs1, Parent, BgSt),
BatchFun = fun({_, #doc{} = Doc}) ->
ok = gen_server:call(Parent, {batch_doc, Doc}, infinity)
end,
lists:foreach(BatchFun, lists:sort(maps:to_list(Docs))),
% Invidually upload docs with attachments.
maps:map(FetchFun, maps:without(maps:keys(Docs), IdRevs1)),
{ok, Stats} = gen_server:call(Parent, flush, infinity),
ok = report_seq_done(Cp, ReportSeq, Stats),
couch_log:debug("Worker reported completion of seq ~p", [ReportSeq]),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,11 @@ bulk_get_test_() ->
fun couch_replicator_test_helper:test_teardown/1,
[
?TDEF_FE(use_bulk_get),
?TDEF_FE(use_bulk_get_with_ddocs),
?TDEF_FE(use_bulk_get_with_attachments),
?TDEF_FE(dont_use_bulk_get),
?TDEF_FE(dont_use_bulk_get_ddocs),
?TDEF_FE(dont_use_bulk_get_attachments),
?TDEF_FE(job_enable_overrides_global_disable),
?TDEF_FE(global_disable_works)
]
Expand All @@ -39,7 +43,33 @@ use_bulk_get({_Ctx, {Source, Target}}) ->
replicate(Source, Target, true),
BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3),
JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6),
DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4),
?assertEqual(0, JustGets),
?assertEqual(0, DocUpdates),
?assert(BulkGets >= 1),
compare_dbs(Source, Target).

use_bulk_get_with_ddocs({_Ctx, {Source, Target}}) ->
populate_db_ddocs(Source, ?DOC_COUNT),
meck:new(couch_replicator_api_wrap, [passthrough]),
replicate(Source, Target, true),
BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3),
JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6),
DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4),
?assertEqual(?DOC_COUNT, JustGets),
?assertEqual(?DOC_COUNT, DocUpdates),
?assert(BulkGets >= 1),
compare_dbs(Source, Target).

use_bulk_get_with_attachments({_Ctx, {Source, Target}}) ->
populate_db_atts(Source, ?DOC_COUNT),
meck:new(couch_replicator_api_wrap, [passthrough]),
replicate(Source, Target, true),
BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3),
JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6),
DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4),
?assertEqual(?DOC_COUNT, JustGets),
?assertEqual(?DOC_COUNT, DocUpdates),
?assert(BulkGets >= 1),
compare_dbs(Source, Target).

Expand All @@ -49,10 +79,36 @@ dont_use_bulk_get({_Ctx, {Source, Target}}) ->
replicate(Source, Target, false),
BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3),
JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6),
DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4),
?assertEqual(0, BulkGets),
?assertEqual(0, DocUpdates),
?assertEqual(?DOC_COUNT, JustGets),
compare_dbs(Source, Target).

dont_use_bulk_get_ddocs({_Ctx, {Source, Target}}) ->
populate_db_ddocs(Source, ?DOC_COUNT),
meck:new(couch_replicator_api_wrap, [passthrough]),
replicate(Source, Target, false),
BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3),
JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6),
DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4),
?assertEqual(0, BulkGets),
?assertEqual(?DOC_COUNT, JustGets),
?assertEqual(?DOC_COUNT, DocUpdates),
compare_dbs(Source, Target).

dont_use_bulk_get_attachments({_Ctx, {Source, Target}}) ->
populate_db_atts(Source, ?DOC_COUNT),
meck:new(couch_replicator_api_wrap, [passthrough]),
replicate(Source, Target, false),
BulkGets = meck:num_calls(couch_replicator_api_wrap, bulk_get, 3),
JustGets = meck:num_calls(couch_replicator_api_wrap, open_doc_revs, 6),
DocUpdates = meck:num_calls(couch_replicator_api_wrap, update_doc, 4),
?assertEqual(0, BulkGets),
?assertEqual(?DOC_COUNT, JustGets),
?assertEqual(?DOC_COUNT, DocUpdates),
compare_dbs(Source, Target).

job_enable_overrides_global_disable({_Ctx, {Source, Target}}) ->
populate_db(Source, ?DOC_COUNT),
Persist = false,
Expand All @@ -78,10 +134,31 @@ global_disable_works({_Ctx, {Source, Target}}) ->
compare_dbs(Source, Target).

populate_db(DbName, DocCount) ->
Fun = fun(Id, Acc) -> [#doc{id = integer_to_binary(Id)} | Acc] end,
IdFun = fun(Id) -> integer_to_binary(Id) end,
Fun = fun(Id, Acc) -> [#doc{id = IdFun(Id)} | Acc] end,
Docs = lists:foldl(Fun, [], lists:seq(1, DocCount)),
{ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).

populate_db_ddocs(DbName, DocCount) ->
IdFun = fun(Id) -> <<"_design/", (integer_to_binary(Id))/binary>> end,
Fun = fun(Id, Acc) -> [#doc{id = IdFun(Id)} | Acc] end,
Docs = lists:foldl(Fun, [], lists:seq(1, DocCount)),
{ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).

populate_db_atts(DbName, DocCount) ->
IdFun = fun(Id) -> integer_to_binary(Id) end,
Fun = fun(Id, Acc) -> [#doc{id = IdFun(Id), atts = [att(<<"a">>)]} | Acc] end,
Docs = lists:foldl(Fun, [], lists:seq(1, DocCount)),
{ok, _} = fabric:update_docs(DbName, Docs, [?ADMIN_CTX]).

att(Name) when is_binary(Name) ->
couch_att:new([
{name, Name},
{att_len, 1},
{type, <<"app/binary">>},
{data, <<"x">>}
]).

compare_dbs(Source, Target) ->
couch_replicator_test_helper:cluster_compare_dbs(Source, Target).

Expand Down
4 changes: 2 additions & 2 deletions src/docs/src/api/database/find.rst
Original file line number Diff line number Diff line change
Expand Up @@ -705,8 +705,8 @@ operators require the argument to be in a specific JSON format.
| | | | document. Non-array fields cannot |
| | | | match this condition. |
+---------------+-------------+------------+-----------------------------------+
| Miscellaneous | ``$mod`` | [Divisor, | Divisor and Remainder are both |
| | | Remainder] | positive or negative integers. |
| Miscellaneous | ``$mod`` | [Divisor, | Divisor is a non-zero integer, |
| | | Remainder] | Remainder is any integer. |
| | | | Non-integer values result in a |
| | | | 404. Matches documents where |
| | | | ``field % Divisor == Remainder`` |
Expand Down
2 changes: 1 addition & 1 deletion src/mango/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -302,7 +302,7 @@ Array related operators

Misc related operators

* "$mod" - [Divisor, Remainder], where Divisor and Remainder are both positive integers (ie, greater than 0). Matches documents where (field % Divisor == Remainder) is true. This is false for any non-integer field
* "$mod" - [Divisor, Remainder], where Divisor is a non-zero integer and Remainder is any integer. Matches documents where (field % Divisor == Remainder) is true. This is false for any non-integer field
* "$regex" - string, a regular expression pattern to match against the document field. Only matches when the field is a string value and matches the supplied matches


Expand Down

0 comments on commit a9be938

Please sign in to comment.