Update images and reduce their size (#13)

* Allow container to be run with custom command

* Reduce language image sizes

* Update OpenJDK to latest version

* Use swc for TypeScript and don't print by default

* Update Haskell image and use stable Debian slim

* Fix formatting and revert TS changes

.github/workflows/build.yaml

@@ -0,0 +1,58 @@
+name: Builds
+
+on:
+  release:
+    types: [created]
+
+jobs:
+  build:
+    strategy:
+      fail-fast: false
+      matrix:
+        ghc: ['8.8.3']
+        os: [ubuntu-latest, macos-latest]
+        include: # GHC 8.8.3 fails to install on Windows
+          - ghc: '8.6.5'
+            os: windows-latest
+
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-haskell@v1.1.1
+        with:
+          ghc-version: ${{ matrix.ghc }}
+          cabal-version: '3.2'
+
+      # - name: Freeze
+      #   run: cabal freeze
+
+      # - name: Cache Cabal
+      #   uses: actions/cache@v1.2.0
+      #   with:
+      #     path: ${{ steps.setup-haskell-cabal.outputs.cabal-store }}
+      #     key: ${{ runner.OS }}-${{ matrix.ghc }}-${{ hashFiles('cabal.project.freeze') }}
+
+      - name: Build Myriad
+        run: cabal build -O2 myriad:exe:myriad
+
+      - name: Find Binary
+        id: find_binary
+        shell: bash
+        run: |
+          FOUND=$(find dist-newstyle \( -name 'myriad' -o -name 'myriad.exe' \) -type f)
+          cp $FOUND myriad
+          cp config.example.yaml config.yaml
+          strip myriad
+          tar -cvzf myriad-${{ github.event.release.name }}-${{ runner.OS }}-${{ matrix.ghc }}.tar.gz config.yaml languages myriad
+
+      - name: Upload Binary
+        uses: actions/upload-release-asset@v1.0.2
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          upload_url: ${{ github.event.release.upload_url }}
+          asset_path: myriad-${{ github.event.release.name }}-${{ runner.OS }}-${{ matrix.ghc }}.tar.gz
+          asset_name: myriad-${{ github.event.release.name }}-${{ runner.OS }}-${{ matrix.ghc }}.tar.gz
+          asset_content_type: application/gzip

Dockerfile

@@ -0,0 +1,37 @@
+FROM alpine:latest as build
+
+ENV LANG     en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL   en_US.UTF-8
+
+WORKDIR /tmp/haskell
+RUN apk update && \
+    apk upgrade --available && \
+    apk add \
+        build-base make cmake gcc gmp curl xz perl cpio coreutils \
+        binutils-gold tar gzip unzip \
+        libc-dev musl-dev ncurses-dev gmp-dev zlib-dev expat-dev libffi-dev \
+        gd-dev postgresql-dev linux-headers
+
+RUN curl https://gitlab.haskell.org/haskell/ghcup-hs/raw/master/bootstrap-haskell -sSf | BOOTSTRAP_HASKELL_NONINTERACTIVE=1 sh && \
+    /root/.ghcup/bin/ghcup set
+ENV PATH "$PATH:/root/.cabal/bin:/root/.ghcup/bin"
+
+WORKDIR /tmp/myriad
+COPY . .
+RUN cabal new-install
+
+RUN mkdir -p /opt/myriad && \
+    cp -L /root/.cabal/bin/myriad /opt/myriad && \
+    mv languages /opt/myriad && \
+    mv config.example.yaml /opt/myriad/config.yaml
+
+
+FROM alpine:latest
+RUN apk add --no-cache docker-cli gmp
+WORKDIR /opt/myriad
+COPY --from=build /opt/myriad .
+
+EXPOSE 8081
+
+ENTRYPOINT ["./myriad"]

Dockerfile.release

@@ -0,0 +1,19 @@
+FROM frolvlad/alpine-glibc:latest as build
+
+ARG MYRIAD_VERSION=0.5.0.3
+ARG GHC_VERSION=8.8.3
+
+RUN apk add --no-cache curl tar gzip
+WORKDIR /tmp/myriad
+RUN curl -OL https://github.com/1Computer1/myriad/releases/download/${MYRIAD_VERSION}/myriad-${MYRIAD_VERSION}-Linux-${GHC_VERSION}.tar.gz && \
+    tar -xzf myriad-${MYRIAD_VERSION}-Linux-${GHC_VERSION}.tar.gz && \
+    rm -f myriad-${MYRIAD_VERSION}-Linux-${GHC_VERSION}.tar.gz
+
+FROM frolvlad/alpine-glibc:latest
+RUN apk add --no-cache docker-cli gmp
+WORKDIR /opt/myriad
+COPY --from=build /tmp/myriad .
+
+EXPOSE 8081
+
+ENTRYPOINT ["./myriad"]

README.md

@@ -14,6 +14,7 @@ Features include:
     - Maximum evaluation time.
     - Maximum concurrent evaluations.
     - Maximum number of retries.
+    - Maximum output size.
 
 Requires Docker 18+ to operate.  
 

config.example.yaml

@@ -1,3 +1,42 @@
+# Whether to build images concurrently.
+# This will take up more resources when building all the images for the first time.
+buildConcurrently: true
+
+# Whether to start containers on startup of myriad.
+prepareContainers: false
+
+# Interval in minutes to kill all running languages containers.
+cleanupInterval: 30
+
+# Port to run myriad on.
+port: 8081
+
+# The default language configuration.
+defaultLanguage:
+    # The OCI runtime to use when running the container.
+    runtime: runc
+
+    # The maximum memory and swap usage (separately) of a container.
+    memory: 256m
+
+    # The number of CPUs to use.
+    cpus: 0.25
+
+    # Time in seconds for an evaluation before the container kills itself.
+    timeout: 20
+
+    # The maximum number of concurrent evaluations in the container.
+    concurrent: 5
+
+    # The maximum number of retries when the evaluation fails due to a non-timeout related reason.
+    retries: 10
+
+    # The maximum number of bytes that can be outputted.
+    outputLimit: 4k
+
+# The languages to enable.
+# The fields available are the same as in 'defaultLanguage', plus the name of the language.
+# The names are as in your 'languages' folder.
 languages:
     - name: apl
     - name: bash
@@ -27,15 +66,3 @@ languages:
     - name: ruby
     - name: rust
     - name: typescript
-
-defaultLanguage:
-    memory: 256m
-    cpus: 0.25
-    timeout: 20
-    concurrent: 10
-    retries: 2
-
-buildConcurrently: true
-prepareContainers: false
-cleanupInterval: 30
-port: 8081

docker-compose.example.yaml

@@ -0,0 +1,17 @@
+version: '3.8'
+
+services:
+  myriad:
+    build:
+      context: .
+      dockerfile: Dockerfile.release
+    image: myriad:latest
+    container_name: myriad
+    network_mode: bridge
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - ./config.yaml:/opt/myriad/config.yaml:ro
+    ports:
+      - 127.0.0.1:8081:8081/tcp
+      - ::1:8081:8081/tcp
+    restart: unless-stopped

languages/brainfuck/Dockerfile

@@ -1,12 +1,12 @@
 FROM alpine AS build
 
-RUN apk update && apk add g++
 COPY bf.cpp .
-RUN g++ bf.cpp -o bf
+RUN apk add --no-cache g++ && \
+    g++ bf.cpp -o bf
 
 FROM alpine
 LABEL author="1Computer1"
 
-RUN apk update && apk add libstdc++
+RUN apk add --no-cache libstdc++
 COPY --from=build bf /usr/local/bin/
 COPY run.sh /var/run/

languages/c/Dockerfile

@@ -1,7 +1,6 @@
 FROM alpine
 LABEL author="1Computer1"
 
-RUN apk update
-RUN apk add gcc libc-dev
+RUN apk add --no-cache gcc libc-dev
 
 COPY run.sh /var/run/

languages/cpp/Dockerfile

@@ -1,7 +1,6 @@
 FROM alpine
 LABEL author="1Computer1"
 
-RUN apk update
-RUN apk add g++
+RUN apk add --no-cache g++
 
 COPY run.sh /var/run/

languages/haskell/Dockerfile

@@ -1,14 +1,17 @@
-FROM debian:stretch
+FROM debian:stable-slim
 LABEL author="1Computer1"
 ENV LANG C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends gnupg dirmngr && \
-    echo 'deb http://downloads.haskell.org/debian stretch main' > /etc/apt/sources.list.d/ghc.list && \
+    apt-get install -y --no-install-recommends gnupg dirmngr ca-certificates && \
+    echo 'deb https://downloads.haskell.org/debian stretch main' > /etc/apt/sources.list.d/ghc.list && \
     apt-key adv --keyserver keyserver.ubuntu.com --recv-keys BA3CBA3FFE22B574 && \
     apt-get update && \
-    apt-get install -y --no-install-recommends ghc-8.6.5
+    apt-get install -y --no-install-recommends ghc-9.0.1 && \
+    apt-get purge -y gnupg dirmngr ca-certificates && \
+    apt-get autoremove -y && \
+    apt-get autoclean -y
 
-ENV PATH /opt/ghc/8.6.5/bin:$PATH
+ENV PATH /opt/ghc/9.0.1/bin:$PATH
 
 COPY run.sh /var/run/

languages/idris/Dockerfile

@@ -2,6 +2,7 @@ FROM alpine:latest
 
 RUN echo "@testing http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \
     apk update && \
-    apk add idris@testing
+    apk add idris@testing && \
+    rm -rf /var/cache/apk/*
 
 COPY run.sh /var/run/

languages/java/Dockerfile

@@ -1,4 +1,4 @@
-FROM openjdk:13-alpine
+FROM openjdk:17-alpine
 LABEL author="1Computer1"
 
 COPY run.sh /var/run/

languages/javascript/run.sh

@@ -1 +1 @@
-cat | node -p
+cat | node

languages/lua/Dockerfile

@@ -1,6 +1,5 @@
 FROM alpine
 
-RUN apk update
-RUN apk add lua5.3
+RUN apk add --no-cache lua5.3
 
 COPY run.sh /var/run/

languages/nim/run.sh

@@ -1,2 +1,2 @@
 cat > program.nim
-nim compile --run --colors=off --memTracker=off --verbosity=0 --hints=off --nimcache:/tmp/"$CODEDIR"/cache ./program.nim
+nim compile --run --colors=off --memTracker=off --verbosity=0 --hints=off --warnings=off --nimcache:/tmp/"$CODEDIR"/cache ./program.nim

languages/typescript/run.sh

@@ -1,4 +1,4 @@
 cat > program.ts
 tsc --lib DOM,ESNext --target ES2019 --strict \
     --skipLibCheck --types /usr/local/share/.config/yarn/global/node_modules/@types/node program.ts \
-    && cat program.js | node -p 
+    && cat program.js | node

myriad.cabal

@@ -1,7 +1,7 @@
 cabal-version:  2.2
 
 name:           myriad
-version:        0.4.1.0
+version:        0.5.0.3
 synopsis:       Arbitrary code execution in Docker.
 description:    Please see the README on GitHub at <https://github.com/1Computer1/myriad#readme>
 category:       Server
@@ -46,7 +46,7 @@ common shared
   build-depends:
       aeson
     , async
-    , base >= 4.13 && < 5
+    , base >= 4.12 && < 5
     , bytestring
     , containers
     , filepath

src/Myriad/Config.hs

@@ -21,11 +21,13 @@ type LanguageName = T.Text
 
 data Language = Language
     { _name :: LanguageName
+    , _runtime :: T.Text
     , _memory :: T.Text
     , _cpus :: Double
     , _timeout :: Int
     , _concurrent :: Int
     , _retries :: Int
+    , _outputLimit :: T.Text
     } deriving (Show)
 
 makeFieldLabelsWith classUnderscoreNoPrefixFields ''Language
@@ -41,30 +43,36 @@ data Config = Config
 makeFieldLabelsWith classUnderscoreNoPrefixFields ''Config
 
 data DefaultLanguage = DefaultLanguage
-    { _memory :: T.Text
+    { _runtime :: T.Text
+    , _memory :: T.Text
     , _cpus :: Double
     , _timeout :: Int
     , _concurrent :: Int
     , _retries :: Int
+    , _outputLimit :: T.Text
     } deriving (Show)
 
 makeFieldLabelsWith classUnderscoreNoPrefixFields ''DefaultLanguage
 
 instance FromJSON DefaultLanguage where
     parseJSON = withObject "default language" $ \m -> DefaultLanguage
-        <$> m .: "memory"
+        <$> m .: "runtime"
+        <*> m .: "memory"
         <*> m .: "cpus"
         <*> m .: "timeout"
         <*> m .: "concurrent"
         <*> m .: "retries"
+        <*> m .: "outputLimit"
 
 data RawLanguage = RawLanguage
     { _name :: LanguageName
+    , _runtime :: Maybe T.Text
     , _memory :: Maybe T.Text
     , _cpus :: Maybe Double
     , _timeout :: Maybe Int
     , _concurrent :: Maybe Int
     , _retries :: Maybe Int
+    , _outputLimit :: Maybe T.Text
     } deriving (Show)
 
 makeFieldLabelsWith classUnderscoreNoPrefixFields ''RawLanguage
@@ -72,11 +80,13 @@ makeFieldLabelsWith classUnderscoreNoPrefixFields ''RawLanguage
 instance FromJSON RawLanguage where
     parseJSON = withObject "language" $ \m -> RawLanguage
         <$> m .: "name"
+        <*> m .:? "runtime"
         <*> m .:? "memory"
         <*> m .:? "cpus"
         <*> m .:? "timeout"
         <*> m .:? "concurrent"
         <*> m .:? "retries"
+        <*> m .:? "outputLimit"
 
 data RawConfig = RawConfig
     { _languages :: [RawLanguage]
@@ -122,9 +132,11 @@ fromRawLanguage :: DefaultLanguage -> RawLanguage -> Language
 fromRawLanguage d r =
     Language
         { _name = r ^. #name
+        , _runtime = fromMaybe (d ^. #runtime) (r ^. #runtime)
         , _memory = fromMaybe (d ^. #memory) (r ^. #memory)
         , _cpus = fromMaybe (d ^. #cpus) (r ^. #cpus)
         , _timeout = fromMaybe (d ^. #timeout) (r ^. #timeout)
         , _concurrent = fromMaybe (d ^. #concurrent) (r ^. #concurrent)
         , _retries = fromMaybe (d ^. #retries) (r ^. #retries)
+        , _outputLimit = fromMaybe (d ^. #outputLimit) (r ^. #outputLimit)
         }

src/Myriad/Docker.hs

@@ -40,13 +40,19 @@ data EvalResult
 buildImage :: Language -> Myriad ()
 buildImage lang = do
     env <- ask
-    logInfo ["Building image ", cs $ imageName lang]
-    exec_ ["docker build -t ", imageName lang, " ", cs (env ^. #languagesDir) </> cs (lang ^. #name)]
-    setupQSems
-    logInfo ["Built image ", cs $ imageName lang]
-    when (env ^. #config % #prepareContainers) . void $ setupContainer lang
+    logInfo ["Checking for image ", cs $ imageName lang]
+    res <- try $ exec ["docker images -q ", imageName lang]
+    case res of
+        Left (SomeException err) -> logError ["An exception occured when checking for image ", cs $ imageName lang, ":\n", cs $ show err]
+        Right s -> do
+            when (BL.null s) . void $ do -- If string is empty that means the image does not yet exist
+                logInfo ["Building image ", cs $ imageName lang]
+                exec_ ["docker build -t ", imageName lang, " ", cs (env ^. #languagesDir) </> cs (lang ^. #name)]
+                logInfo ["Built image ", cs $ imageName lang]
+            setupQSems
+            when (env ^. #config % #prepareContainers) . void $ setupContainer lang
     where
-        setupQSems ::  Myriad ()
+        setupQSems :: Myriad ()
         setupQSems = do
             env <- ask
             csem <- newQSem 1 -- We only want one container to be set up at a time
@@ -69,15 +75,17 @@ startCleanup :: Myriad ()
 startCleanup = do
     config <- gview #config
     when (config ^. #cleanupInterval > 0) . void $ do
-        let t = fromIntegral (config ^. #cleanupInterval) * 60000000
+        let t = fromIntegral (config ^. #cleanupInterval)
         fork $ timer t
     where
+        -- Given time in minutes
         timer :: Int -> Myriad ()
         timer t = forever $ do
-            threadDelay t
+            -- Takes time in microseconds
+            threadDelay $ t  * 60000000
             logInfo ["Starting cleanup of containers"]
             n <- killContainers
-            logInfo ["Cleaned up ", cs $ show n, " containers, next in ", cs $ show t, " seconds"]
+            logInfo ["Cleaned up ", cs $ show n, " containers, next in ", cs $ show t, " minutes"]
             timer t
 
 setupContainer :: Language -> Myriad ContainerName
@@ -93,7 +101,9 @@ setupContainer lang = do
             cnt <- newContainerName lang
             logInfo ["Setting up new container ", cs cnt]
             exec_
-                [ "docker run --rm --name="
+                [ "docker run --runtime="
+                , cs $ lang ^. #runtime
+                , " --rm --name="
                 , cs cnt
                 -- User 1000 will be for setting up the environment
                 , " -u1000:1000 -w/tmp/ -dt --net=none --cpus="
@@ -202,8 +212,11 @@ evalCode lang numRetries code = withContainer $ \cnt -> do
 
         timer :: ContainerName -> MVar Bool -> Myriad ()
         timer cnt doneRef = do
-            logDebug ["Starting timeout of ", cs . show $ lang ^. #timeout, " seconds for container ", cs cnt]
-            threadDelay $ fromIntegral (lang ^. #timeout) * 1000000
+            -- Given time in seconds
+            let t = fromIntegral $ lang ^. #timeout
+            logDebug ["Starting timeout of ", cs . show $ t, " seconds for container ", cs cnt]
+            -- Takes time in microseconds
+            threadDelay $ t * 1000000
             done <- readMVar doneRef
             if done
                 then do
@@ -214,17 +227,19 @@ evalCode lang numRetries code = withContainer $ \cnt -> do
                     void . killContainer $ lang ^. #name
 
         eval :: ContainerName -> Snowflake -> Myriad EvalResult
-        eval cnt snowflake = do
+        eval cnt snowflake = do 
             logInfo ["Running code in container ", cs cnt, ", evaluation ", cs $ show snowflake, ":\n", cs code]
             exec_ ["docker exec ", cs cnt, " mkdir eval/", show snowflake]
             exec_ ["docker exec ", cs cnt, " chmod 777 eval/", show snowflake]
             -- User 1001 will be used for the actual execution so that they can't access `eval` itself
-            let cmd = mconcat
+            let limit = lang ^. #outputLimit
+                cmd = mconcat
                         [ "docker exec -i -u1001:1001 -w/tmp/eval/"
                         , show snowflake
                         , " "
                         , cnt
-                        , " /bin/sh /var/run/run.sh | head -c 4K"
+                        , " /bin/sh /var/run/run.sh 2>&1 | head -c "
+                        , cs limit
                         ]
                 pr = setStdin (byteStringInput $ cs code) $ shell cmd
             logDebug ["Executing with stdin `", cs cmd, "`"]

stack.yaml

@@ -1,4 +1,4 @@
-resolver: lts-15.15
+resolver: lts-16.2
 
 packages:
 - .