Fix #185: Support and use autogen-modules. (#196)

* Fix #185: Support and use autogen-modules.

This makes it possible to generate tarballs without generated modules
(which would be regenerated anyway when the package is build).
See tensorflow/haskell#180 for an example of the issues that causes.

For `Cabal-1.*`, this continues the behavior as before.

Unfortunately, `hpack` requires `cabal-version: >=2.0` when you use
its `generated-modules` or `generated-other-modules` fields.
Our current set of LTSes that we support still includes `Cabal-1.*`
(which I think is correct).  Luckily we could work around that
using `hpack`'s `verbatim` field to accomplish the same thing a little
more verbosely.  Additionally, I mitigated the situation a little by changing
the Cabal test script to not `sdist` packages that we're not releasing
(`proto-lens-{tests/benchmarks}`).

This change required bumping `stack` to `1.7.1` in order to get new enough
versions of `Cabal` and `hpack`.  Happily, it greatly simplifies
the steps for releasing our packages.

* More README updates

.travis.yml

@@ -36,7 +36,7 @@ before_install:
       cabal)
         export PATH=/opt/ghc/$GHCVER/bin:/opt/cabal/$CABALVER/bin:$HOME/.local/bin:$HOME/.cabal/bin:$PATH;;
     esac
-  - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.6.1/stack-1.6.1-linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C $HOME/.local/bin '*/stack'
+  - curl -L https://github.com/commercialhaskell/stack/releases/download/v1.7.1/stack-1.7.1-linux-x86_64.tar.gz | tar xz --wildcards --strip-components=1 -C $HOME/.local/bin '*/stack'
   - curl -L https://github.com/google/protobuf/releases/download/v3.0.0-beta-2/protoc-3.0.0-beta-2-linux-x86_64.zip > protoc-release.zip
   - unzip -p protoc-release.zip protoc > $HOME/.local/bin/protoc
   - chmod a+x $HOME/.local/bin/protoc

README.md

@@ -26,6 +26,8 @@ To build and test this repository from HEAD, run:
     git submodule update --init --recursive
     stack test
 
+Note: building this repository requires `stack-1.7.1` or newer.
+
 ## Using in a Cabal or Stack package
 `proto-lens` is available on Hackage and Stackage.  Cabal and Stack projects can use it
 to auto-generate Haskell source files from the original
@@ -56,6 +58,7 @@ For example, in `foo-bar-proto.cabal`:
 
     library
         exposed-modules: Proto.Foo.Bar, Proto.Foo.Bar_Fields
+        autogen-modules: Proto.Foo.Bar, Proto.Foo.Bar_Fields
         build-depends: proto-lens-protoc, ...
 
 Next, write a `Setup.hs` file that uses `Data.ProtoLens.Setup` and specifies the

docs/releasing.md

@@ -17,26 +17,8 @@ since they are intended only for use within this repository:
 - Make sure you are on a commit that has been merged upstream to `google/proto-lens`.
 
 ## Steps
-1. For each package *without* autogenerated proto modules, simply run (from the root of the repo):
+First, run `stack build` at the top level.  Then, for each package that you want to upload, run
 
         stack upload {name-of-package}
 
-  Then verify that they've been uploaded successfully by visiting `hackage.org` in a web browser.
-
-2. For packages *with* autogenerated proto-modules (currently:
-`proto-lens-protobuf-types` and `proto-lens-combinators`): these are a little
-harder to upload due to #185.  We have a script for running `sdist`:
-
-        ./release.sh
-
-  Then, for each package that should be uploaded:
-
-        cabal upload {package}/dist/{path}.tar.gz
-
-3. Finally, tag each uploaded package with its version and push it back to upstream.  For example:
-
-        git tag -a proto-lens-v0.3.0.0 -m "Release proto-lens v0.3.0.0"
-        ...
-        git remote add upstream [email protected]:google/proto-lens.git
-        git push upstream --follow-tags  # See: https://stackoverflow.com/a/26438076
-
+Then verify that they've been uploaded successfully by visiting `hackage.org` in a web browser.

proto-lens-combinators/package.yaml

@@ -48,3 +48,10 @@ tests:
     other-modules:
       Proto.Combinators
       Proto.Combinators_Fields
+    # Manually list autogenerated modules, to avoid hpack
+    # requiring `cabal-version: >= 2.0`.
+    verbatim:
+      - |
+        autogen-modules:
+          Proto.Combinators
+          Proto.Combinators_Fields

proto-lens-discrimination/package.yaml

@@ -54,3 +54,10 @@ tests:
     other-modules:
       - Proto.Enum
       - Proto.Enum_Fields
+    # Manually list autogenerated modules, to avoid hpack
+    # requiring `cabal-version: >= 2.0`.
+    verbatim:
+      - |
+        autogen-modules:
+          Proto.Enum
+          Proto.Enum_Fields

proto-lens-protobuf-types/package.yaml

@@ -42,3 +42,16 @@ library:
     - Proto.Google.Protobuf.Wrappers_Fields
     - Proto.Google.Protobuf.Timestamp
     - Proto.Google.Protobuf.Timestamp_Fields
+  # Manually list autogenerated modules, to avoid hpack
+  # requiring `cabal-version: >= 2.0`.
+  verbatim:
+    - |
+      autogen-modules:
+        Proto.Google.Protobuf.Any
+        Proto.Google.Protobuf.Any_Fields
+        Proto.Google.Protobuf.Duration
+        Proto.Google.Protobuf.Duration_Fields
+        Proto.Google.Protobuf.Wrappers
+        Proto.Google.Protobuf.Wrappers_Fields
+        Proto.Google.Protobuf.Timestamp
+        Proto.Google.Protobuf.Timestamp_Fields

proto-lens-protoc/src/Data/ProtoLens/Setup.hs

@@ -185,11 +185,16 @@ generatingSpecificProtos root getProtos hooks = hooks
     { buildHook = \p l h f -> generate l >> buildHook hooks p l h f
     , haddockHook = \p l h f -> generate l >> haddockHook hooks p l h f
     , replHook = \p l h f args -> generate l >> replHook hooks p l h f args
+#if !MIN_VERSION_Cabal(2,0,0)
+    -- Older versions of Cabal don't support the autogen-modules field.
+    -- Work around it by manually generating the modules and putting them
+    -- in a place where `cabal sdist` will pick them up.
     , sDistHook = \p maybe_l h f -> case maybe_l of
             Nothing -> error "Can't run protoc; run 'cabal configure' first."
             Just l -> do
                         generate l
                         sDistHook hooks (fudgePackageDesc l p) maybe_l h f
+#endif
     , postCopy = \a flags pkg lbi -> do
                   let verb = fromFlag $ copyVerbosity flags
                   let destDir = datadir (absoluteInstallDirs pkg lbi

proto-lens-tests-dep/package.yaml

@@ -32,3 +32,10 @@ library:
   exposed-modules:
     - Proto.TestDep.Foo
     - Proto.TestDep.Foo_Fields
+  # Manually list autogenerated modules, to avoid hpack
+  # requiring `cabal-version: >= 2.0`.
+  verbatim:
+    - |
+      autogen-modules:
+        Proto.TestDep.Foo
+        Proto.TestDep.Foo_Fields

travis-cabal.sh

@@ -12,7 +12,11 @@ chmod +x $HOME/.local/bin/hpack
 # List all the packages in this repo.  Put certain ones first since
 # they're dependencies of the others.  (Unfortunately, "stack query" doesn't
 # give them to us in the right order.)
-PACKAGES="
+
+# All the packages that are prepared to be test-released.
+# Don't include test packages, some of which require
+# Cabal-2.0 or higher during `sdist` due to using hpack.
+PACKAGES_TO_INSTALL="
     discrimination-ieee754
     lens-labels
     proto-lens
@@ -21,12 +25,17 @@ PACKAGES="
     proto-lens-arbitrary
     proto-lens-combinators
     proto-lens-optparse
+    proto-lens-discrimination
     proto-lens-tests-dep
+"
+
+PACKAGES_TO_BUILD="
     proto-lens-tests
-    proto-lens-discrimination
     proto-lens-benchmarks
-"
-echo Building: $PACKAGES
+"""
+
+echo Installing: ${PACKAGES_TO_INSTALL}
+echo Building: ${PACKAGES_TO_BUILD}
 
 # Needed by haskell-src-exts which is a dependency of proto-lens-protoc.
 # Sadly, Cabal won't install such build-tools automatically.
@@ -35,33 +44,44 @@ cabal install happy
 # Unregister the already-installed packages, since otherwise they may
 # propagate between builds.
 # TODO: use a Cabal sandbox for this.
-for p in $PACKAGES
+for p in $PACKAGES_TO_INSTALL
 do
   echo "Unregistering $p"
   ghc-pkg unregister --force $p || true
 done
 
-for p in $PACKAGES
+function build_package () {
+  hpack
+  cabal clean
+  cabal install --enable-tests --force --only-dependencies
+  cabal configure --enable-tests --enable-benchmarks
+  cabal build
+}
+
+
+for p in $PACKAGES_TO_INSTALL
 do
-    echo "Cabal building $p"
+    echo "Cabal installing $p"
     (cd $p &&
-        hpack # Generate the .cabal file
-        cabal clean
-        cabal install --enable-tests --only-dependencies
-        cabal configure --enable-tests --enable-benchmarks
-        cabal build
-        cabal sdist
-        SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz"; exit}')
-        cd dist
-        if [ -f "$SRC_TGZ" ]; then
-            cabal install --force-reinstalls "$SRC_TGZ"
-        else
-            echo "expected '$SRC_TGZ' not found"
-            exit 1
-        fi
+      build_package
+      cabal sdist
+      SRC_TGZ=$(cabal info . | awk '{print $2 ".tar.gz"; exit}')
+      cd dist
+      if [ -f "$SRC_TGZ" ]; then
+          cabal install --force-reinstalls "$SRC_TGZ"
+      else
+          echo "expected '$SRC_TGZ' not found"
+          exit 1
+      fi
     )
 done
 
+for p in $PACKAGES_TO_BUILD
+do
+  echo "Cabal building $p"
+  (cd $p && build_package)
+done
+
 # Check that "cabal repl" works on a simple test.
 cd proto-lens-tests
 # Get rid of the previous dist/autogen to make sure "cabal repl" rebuilds it.