Skip to content

Commit

Permalink
Make Scilla compatible with Zilliqa 2 (#1249)
Browse files Browse the repository at this point in the history
* Make Scilla compatible with Zilliqa 2

* We add another binary which exposes the `scilla-server` JSON-RPC
interface over HTTP.
* The state IPC calls now make requests over HTTP, rather than via
a unix domain socket. Note that this is a breaking change, meaning
this version of Scilla will NOT work with Zilliqa 1.
* All bytes in IPC calls are encoded into base64. Previously, we
would include invalid UTF-8 strings in JSON-RPC requests, which
resulted in invalid JSON.

* Fix test CI

* wip: integration tests

* fix

* add opam installation

* add opam init

* fix typo

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* fix

* fix

* Delete test CI!

---------

Co-authored-by: Mauro Medda <[email protected]>
  • Loading branch information
JamesHinshelwood and mauromedda authored Apr 11, 2024
1 parent 308af35 commit 6fe9206
Show file tree
Hide file tree
Showing 11 changed files with 121 additions and 118 deletions.
63 changes: 0 additions & 63 deletions .github/workflows/ci-image-test.yml

This file was deleted.

3 changes: 2 additions & 1 deletion docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ RUN apt-get update -y \
git \
lcov \
libcurl4-openssl-dev \
libev-dev \
libgmp-dev \
libpcre3-dev \
libssl-dev \
Expand Down Expand Up @@ -119,9 +120,9 @@ FROM ubuntu:22.04

RUN apt-get update -y \
&& apt-get install -y build-essential \
libev-dev \
libgmp-dev

ARG SOURCE_DIR="/scilla/${MAJOR_VERSION}"

COPY --from=builder ${SOURCE_DIR} ${SOURCE_DIR}

17 changes: 0 additions & 17 deletions docker/Dockerfile.test

This file was deleted.

4 changes: 3 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,9 @@
"@opam/ppx_deriving_rpc": ">=6.0.0 <10.0.0",
"@opam/secp256k1": ">=0.4.4 <0.5.0",
"@opam/stdint": ">=0.5.1 <0.8.0",
"@opam/yojson": ">=1.7.0 <2.1.0"
"@opam/yojson": ">=1.7.0 <2.1.0",
"@opam/opium": ">=0.20.0 <1.0.0",
"@opam/ezcurl": ">=0.2.4 <0.3.0"
},
"devDependencies": {
"@opam/merlin": "*",
Expand Down
2 changes: 2 additions & 0 deletions scilla.opam
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ depends: [
"seq"
"stdint" {>= "0.5.1" & < "0.8~"}
"yojson" {>= "1.7.0" & < "2.1~"}
"opium" {>= "0.20.0" & < "1.0.0"}
"ezcurl" {>= "0.2.4" & < "0.3.0"}
]
build: [
[ "./scripts/build_deps.sh" ]
Expand Down
4 changes: 2 additions & 2 deletions src/eval/Disambiguator.ml
Original file line number Diff line number Diff line change
Expand Up @@ -808,7 +808,7 @@ module InputStateService = struct
try
let encoder = Pbrt.Encoder.create () in
Ipcmessage_pb.encode_proto_scilla_query query encoder;
Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder
Base64.encode_exn @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder
with e -> fatal_error (mk_error0 ~kind:(Exn.to_string e) ?inst:None)

let decode_serialized_value value =
Expand Down Expand Up @@ -838,7 +838,7 @@ module InputStateService = struct
in
match res with
| Some (true, res') ->
let decoded_pb = decode_serialized_value (Bytes.of_string res') in
let decoded_pb = decode_serialized_value (Bytes.of_string (Base64.decode_exn res')) in
let res'' = deserialize_value decoded_pb tp this_address in
Some res''
| Some (false, _) | None -> None
Expand Down
58 changes: 32 additions & 26 deletions src/eval/StateIPCClient.ml
Original file line number Diff line number Diff line change
Expand Up @@ -51,26 +51,32 @@ let ipcclient_exn_wrapper thunk =
try thunk () with
| Core_unix.Unix_error (_, s1, s2) ->
fail0 ~kind:("StateIPCClient: Unix error: " ^ s1 ^ s2) ?inst:None
| _ ->
| e ->
let e = Exn.to_string e in
print_endline (Printf.sprintf "error making JSON-RPC call: %s" e);
fail0 ~kind:"StateIPCClient: Unexpected error making JSON-RPC call"
?inst:None

let binary_rpc ~socket_addr (call : Rpc.call) : Rpc.response M.t =
let socket =
Core_unix.socket ~domain:Core_unix.PF_UNIX ~kind:Core_unix.SOCK_STREAM
~protocol:0 ()
in
Core_unix.connect socket ~addr:(Core_unix.ADDR_UNIX socket_addr);
let ic = Core_unix.in_channel_of_descr socket in
let oc = Core_unix.out_channel_of_descr socket in
let http_rpc ~socket_addr (call : Rpc.call) : Rpc.response M.t =
let msg_buf = Jsonrpc.string_of_call ~version:Jsonrpc.V2 call in
DebugMessage.plog (Printf.sprintf "Sending: %s\n" msg_buf);
(* Send data to the socket. *)
let _ = send_delimited oc msg_buf in
(* Get response. *)
let response = Caml.input_line ic in
Core_unix.close socket;
DebugMessage.plog (Printf.sprintf "Response: %s\n" response);
print_endline (Printf.sprintf "Sending: %s\n" msg_buf);
let exception Http_error of string in
let response =
match Ezcurl.post ~headers:["content-type", "application/json"] ~content:(`String msg_buf) ~params:[] ~url:socket_addr () with
| Ok response -> response
| Error (_, err) -> (
print_endline (Printf.sprintf "error calling RPC: %s" err);
raise (Http_error (Printf.sprintf "error calling RPC: %s" err))
)
in

let response = if response.code = 200 then response.body else (
print_endline (Printf.sprintf "error response from RPC: code: %d, body: %s" response.code response.body);
raise (Http_error "error response from RPC")
)
in

print_endline (Printf.sprintf "Response: %s\n" response);
M.return @@ Jsonrpc.response_of_string response

(* Encode a literal into bytes, opaque to the backend storage. *)
Expand Down Expand Up @@ -137,7 +143,7 @@ let encode_serialized_value value =
try
let encoder = Pbrt.Encoder.create () in
Ipcmessage_pb.encode_proto_scilla_val value encoder;
pure @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder
pure @@ Base64.encode_exn @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder
with e -> fail0 ~kind:(Exn.to_string e) ?inst:None

let decode_serialized_value value =
Expand All @@ -150,7 +156,7 @@ let encode_serialized_query query =
try
let encoder = Pbrt.Encoder.create () in
Ipcmessage_pb.encode_proto_scilla_query query encoder;
pure @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder
pure @@ Base64.encode_exn @@ Bytes.to_string @@ Pbrt.Encoder.to_bytes encoder
with e -> fail0 ~kind:(Exn.to_string e) ?inst:None

(* Fetch from a field. "keys" is empty when fetching non-map fields or an entire Map field.
Expand All @@ -168,14 +174,14 @@ let fetch ~socket_addr ~fname ~keys ~tp =
let%bind q' = encode_serialized_query q in
let%bind res =
let thunk () =
translate_res @@ IPCClient.fetch_state_value (binary_rpc ~socket_addr) q'
translate_res @@ IPCClient.fetch_state_value (http_rpc ~socket_addr) q'
in
ipcclient_exn_wrapper thunk
in
match res with
| true, res' ->
let%bind tp' = TypeUtilities.map_access_type tp (List.length keys) in
let%bind decoded_pb = decode_serialized_value (Bytes.of_string res') in
let%bind decoded_pb = decode_serialized_value (Bytes.of_string (Base64.decode_exn res')) in
let%bind res'' = deserialize_value decoded_pb tp' in
pure @@ Some res''
| false, _ -> pure None
Expand Down Expand Up @@ -211,7 +217,7 @@ let external_fetch ~socket_addr ~caddr ~fname ~keys ~ignoreval =
let%bind res =
let thunk () =
translate_res
@@ IPCClient.fetch_ext_state_value (binary_rpc ~socket_addr) caddr q'
@@ IPCClient.fetch_ext_state_value (http_rpc ~socket_addr) caddr q'
in
ipcclient_exn_wrapper thunk
in
Expand All @@ -226,7 +232,7 @@ let external_fetch ~socket_addr ~caddr ~fname ~keys ~ignoreval =
let%bind tp' =
TypeUtilities.map_access_type stored_typ (List.length keys)
in
let%bind decoded_pb = decode_serialized_value (Bytes.of_string res') in
let%bind decoded_pb = decode_serialized_value (Bytes.of_string (Base64.decode_exn res')) in
let%bind res'' = deserialize_value decoded_pb tp' in
pure @@ (Some res'', Some stored_typ)
| false, _, _ -> pure (None, None)
Expand All @@ -247,7 +253,7 @@ let update ~socket_addr ~fname ~keys ~value ~tp =
let%bind () =
let thunk () =
translate_res
@@ IPCClient.update_state_value (binary_rpc ~socket_addr) q' value'
@@ IPCClient.update_state_value (http_rpc ~socket_addr) q' value'
in
ipcclient_exn_wrapper thunk
in
Expand All @@ -267,7 +273,7 @@ let is_member ~socket_addr ~fname ~keys ~tp =
let%bind q' = encode_serialized_query q in
let%bind res =
let thunk () =
translate_res @@ IPCClient.fetch_state_value (binary_rpc ~socket_addr) q'
translate_res @@ IPCClient.fetch_state_value (http_rpc ~socket_addr) q'
in
ipcclient_exn_wrapper thunk
in
Expand All @@ -290,7 +296,7 @@ let remove ~socket_addr ~fname ~keys ~tp =
let%bind () =
let thunk () =
translate_res
@@ IPCClient.update_state_value (binary_rpc ~socket_addr) q' dummy_val
@@ IPCClient.update_state_value (http_rpc ~socket_addr) q' dummy_val
in
ipcclient_exn_wrapper thunk
in
Expand All @@ -304,7 +310,7 @@ let fetch_bcinfo ~socket_addr ~query_name ~query_args =
let%bind res =
let thunk () =
translate_res
@@ IPCClient.fetch_bcinfo (binary_rpc ~socket_addr) query_name query_args
@@ IPCClient.fetch_bcinfo (http_rpc ~socket_addr) query_name query_args
in
ipcclient_exn_wrapper thunk
in
Expand Down
6 changes: 3 additions & 3 deletions src/eval/StateIPCIdl.ml
Original file line number Diff line number Diff line change
Expand Up @@ -69,12 +69,12 @@ module IPCIdl (R : RPC) = struct
let return_update = Param.mk Rpc.Types.unit

let fetch_state_value =
declare "fetchStateValue"
declare "fetchStateValueB64"
[ "Fetch state value from blockchain" ]
(query @-> returning return_fetch RPCError.err)

let fetch_ext_state_value =
declare "fetchExternalStateValue"
declare "fetchExternalStateValueB64"
[ "Fetch state value of another contract from the blockchain" ]
(addr @-> query @-> returning return_ext_fetch RPCError.err)

Expand All @@ -99,7 +99,7 @@ module IPCIdl (R : RPC) = struct
@-> returning return_update RPCError.err)

let update_state_value =
declare "updateStateValue"
declare "updateStateValueB64"
[ "Update state value in blockchain" ]
(query @-> value @-> returning return_update RPCError.err)
end
2 changes: 1 addition & 1 deletion src/eval/dune
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
(wrapped true)
(modes byte native)
(libraries core core_unix.sys_unix angstrom stdint yojson cryptokit
scilla_base rpclib unix rpclib.json rresult ocaml-protoc)
scilla_base rpclib unix rpclib.json rresult ocaml-protoc ezcurl)
(preprocess
(pps ppx_sexp_conv ppx_let bisect_ppx --conditional ppx_deriving_rpc
ppx_deriving.show ppx_compare))
Expand Down
8 changes: 4 additions & 4 deletions src/runners/dune
Original file line number Diff line number Diff line change
@@ -1,14 +1,14 @@
(executables
(names scilla_runner eval_runner type_checker scilla_checker scilla_server
disambiguate_state_json scilla_fmt scilla_merger)
disambiguate_state_json scilla_fmt scilla_merger scilla_server_http)
(public_names scilla-runner eval-runner type-checker scilla-checker
scilla-server disambiguate_state_json scilla-fmt scilla-merger)
scilla-server disambiguate_state_json scilla-fmt scilla-merger scilla-server-http)
(package scilla)
(modules scilla_runner eval_runner type_checker scilla_checker scilla_server
disambiguate_state_json scilla_fmt scilla_merger)
disambiguate_state_json scilla_fmt scilla_merger scilla_server_http)
(libraries core core_unix.command_unix angstrom yojson cryptokit fileutils
scilla_base scilla_eval scilla_server_lib scilla_crypto scilla_format
scilla_merge cmdliner)
scilla_merge cmdliner opium)
(modes byte native)
(preprocess
(pps ppx_sexp_conv ppx_deriving_yojson ppx_let ppx_deriving.show bisect_ppx --conditional)))
Loading

0 comments on commit 6fe9206

Please sign in to comment.