THIS IS A TEST INSTANCE ONLY! REPOSITORIES CAN BE DELETED AT ANY TIME!

Browse Source

Merge remote-tracking branch 'origin/staging' into develop

pull/807/head
Andrey Antukh 1 month ago
parent
commit
8b45ed9c0c
  1. 1
      .gitignore
  2. 81
      backend/scripts/build
  3. 75
      backend/scripts/build.sh
  4. 4
      backend/scripts/import-generic-collections.sh
  5. 19
      backend/scripts/manage.template.sh
  6. 16
      backend/scripts/prepare-release.sh
  7. 2
      backend/scripts/psql.sh
  8. 4
      backend/scripts/run-tests-in-docker.sh
  9. 20
      backend/scripts/run.template.sh
  10. 2
      backend/scripts/smtpd.sh
  11. 2
      backend/scripts/tests.sh
  12. 11
      backend/src/app/config.clj
  13. 3
      docker/devenv/Dockerfile
  14. 1
      docker/devenv/docker-compose.yaml
  15. 2
      docker/devenv/files/nginx.conf
  16. 2
      docker/devenv/files/start-tmux.sh
  17. 1
      exporter/scripts/build
  18. 17
      exporter/scripts/wait-and-start.sh
  19. 17
      frontend/scripts/build
  20. 8
      frontend/scripts/build-and-run-tests.sh
  21. 21
      frontend/scripts/build.sh
  22. 9
      frontend/src/app/main/ui/workspace/viewport/hooks.cljs
  23. 4
      frontend/src/app/main/worker.cljs
  24. 73
      frontend/src/app/util/worker.cljs
  25. 76
      frontend/src/app/worker.cljs
  26. 38
      manage.sh

1
.gitignore

@ -15,6 +15,7 @@ node_modules
/backend/target/
/backend/resources/public/media
/backend/resources/public/assets
/backend/assets/
/backend/dist/
/backend/logs/
/backend/-

81
backend/scripts/build

@ -0,0 +1,81 @@
#!/usr/bin/env bb
;; This Source Code Form is subject to the terms of the Mozilla Public
;; License, v. 2.0. If a copy of the MPL was not distributed with this
;; file, You can obtain one at http://mozilla.org/MPL/2.0/.
;;
;; This Source Code Form is "Incompatible With Secondary Licenses", as
;; defined by the Mozilla Public License, v. 2.0.
;;
;; Copyright (c) UXBOX Labs SL
(ns build
(:require
[clojure.string :as str]
[clojure.java.io :as io]
[clojure.pprint :refer [pprint]]
[babashka.fs :as fs]
[babashka.process :refer [$ check]]))
(defn split-cp
[data]
(str/split data #":"))
(def classpath
(->> ($ clojure -Spath)
(check)
(:out)
(slurp)
(split-cp)
(map str/trim)))
(def classpath-jars
(let [xfm (filter #(str/ends-with? % ".jar"))]
(into #{} xfm classpath)))
(def classpath-paths
(let [xfm (comp (remove #(str/ends-with? % ".jar"))
(filter #(.isDirectory (io/file %))))]
(into #{} xfm classpath)))
(def version
(or (first *command-line-args*) "%version%"))
;; Clean previous dist
(-> ($ rm -rf "./target/dist") check)
;; Create a new dist
(-> ($ mkdir -p "./target/dist/deps") check)
;; Copy all jar deps into dist
(run! (fn [item] (-> ($ cp ~item "./target/dist/deps/") check)) classpath-jars)
;; Create the application jar
(spit "./target/dist/version.txt" version)
(-> ($ jar cvf "./target/dist/deps/app.jar" -C ~(first classpath-paths) ".") check)
(-> ($ jar uvf "./target/dist/deps/app.jar" -C "./target/dist" "version.txt") check)
(run! (fn [item]
(-> ($ jar uvf "./target/dist/deps/app.jar" -C ~item ".") check))
(rest classpath-paths))
;; Copy logging configuration
(-> ($ cp "./resources/log4j2.xml" "./target/dist/") check)
;; Create classpath file
(let [jars (->> (into ["app.jar"] classpath-jars)
(map fs/file-name)
(map #(fs/path "deps" %))
(map str))]
(spit "./target/dist/classpath" (str/join ":" jars)))
;; Copy run script template
(-> ($ cp "./scripts/run.template.sh" "./target/dist/run.sh") check)
;; Copy run script template
(-> ($ cp "./scripts/manage.template.sh" "./target/dist/manage.sh") check)
;; Add exec permisions to scripts.
(-> ($ chmod +x "./target/dist/run.sh") check)
(-> ($ chmod +x "./target/dist/manage.sh") check)
nil

75
backend/scripts/build.sh

@ -1,75 +0,0 @@
#!/usr/bin/env bash
CLASSPATH=`(clojure -Spath)`
NEWCP="./main:./common"
rm -rf ./target/dist
mkdir -p ./target/dist/deps
for item in $(echo $CLASSPATH | tr ":" "\n"); do
if [ "${item: -4}" == ".jar" ]; then
cp $item ./target/dist/deps/;
BN="$(basename -- $item)"
NEWCP+=":./deps/$BN"
fi
done
cp ./resources/log4j2.xml ./target/dist/log4j2.xml
cp -r ./src ./target/dist/main
cp -r ./resources/emails ./target/dist/main/
cp -r ./resources/error-report.tmpl ./target/dist/main/
cp -r ../common ./target/dist/common
echo $NEWCP > ./target/dist/classpath;
tee -a ./target/dist/run.sh >> /dev/null <<EOF
#!/usr/bin/env bash
CP="$NEWCP"
set +e
JAVA_CMD=\$(type -p java)
set -e
if [[ ! -n "\$JAVA_CMD" ]]; then
if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
JAVA_CMD="\$JAVA_HOME/bin/java"
else
>&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
exit 1
fi
fi
if [ -f ./environ ]; then
source ./environ
fi
set -x
exec \$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j2.configurationFile=./log4j2.xml "\$@" clojure.main -m app.main
EOF
tee -a ./target/dist/manage.sh >> /dev/null <<EOF
#!/usr/bin/env bash
CP="$NEWCP"
set +e
JAVA_CMD=\$(type -p java)
set -e
if [[ ! -n "\$JAVA_CMD" ]]; then
if [[ -n "\$JAVA_HOME" ]] && [[ -x "\$JAVA_HOME/bin/java" ]]; then
JAVA_CMD="\$JAVA_HOME/bin/java"
else
>&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
exit 1
fi
fi
if [ -f ./environ ]; then
source ./environ
fi
exec \$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "\$@"
EOF
chmod +x ./target/dist/run.sh
chmod +x ./target/dist/manage.sh

4
backend/scripts/import-generic-collections.sh

@ -1,4 +0,0 @@
#!/usr/bin/env bash
clojure -Adev -m app.cli.collimp $@

19
backend/scripts/manage.template.sh

@ -0,0 +1,19 @@
#!/usr/bin/env bash
set +e
JAVA_CMD=$(type -p java)
set -e
if [[ ! -n "$JAVA_CMD" ]]; then
if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
JAVA_CMD="$JAVA_HOME/bin/java"
else
>&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
exit 1
fi
fi
if [ -f ./environ ]; then
source ./environ
fi
exec $JAVA_CMD $JVM_OPTS -classpath $(cat classpath) -Dlog4j2.configurationFile=./log4j2.xml clojure.main -m app.cli.manage "\$@"

16
backend/scripts/prepare-release.sh

@ -1,16 +0,0 @@
#!/usr/bin/env bash
if [ "$#" -e 0 ]; then
echo "Expecting parameters: 1=path to backend; 2=destination directory"
exit 1
fi
rm -rf $2 || exit 1;
rsync -avr \
--exclude="/test" \
--exclude="/resources/public/media" \
--exclude="/target" \
--exclude="/scripts" \
--exclude="/.*" \
$1 $2;

2
backend/scripts/psql.sh

@ -1,2 +0,0 @@
#!/usr/bin/env bash
PGPASSWORD=$PENPOT_DATABASE_PASSWORD psql $PENPOT_DATABASE_URI -U $PENPOT_DATABASE_USERNAME

4
backend/scripts/run-tests-in-docker.sh

@ -1,4 +0,0 @@
#!/usr/bin/env bash
set -xe
clojure -Adev -m app.tests.main;

20
backend/scripts/run.template.sh

@ -0,0 +1,20 @@
#!/usr/bin/env bash
set +e
JAVA_CMD=$(type -p java)
set -e
if [[ ! -n "$JAVA_CMD" ]]; then
if [[ -n "$JAVA_HOME" ]] && [[ -x "$JAVA_HOME/bin/java" ]]; then
JAVA_CMD="$JAVA_HOME/bin/java"
else
>&2 echo "Couldn't find 'java'. Please set JAVA_HOME."
exit 1
fi
fi
if [ -f ./environ ]; then
source ./environ
fi
set -x
exec $JAVA_CMD $JVM_OPTS -classpath "$(cat classpath)" -Dlog4j2.configurationFile=./log4j2.xml "$@" clojure.main -m app.main

2
backend/scripts/smtpd.sh

@ -1,2 +0,0 @@
#!/usr/bin/env bash
python -m smtpd -n -c DebuggingServer localhost:25

2
backend/scripts/tests.sh

@ -1,2 +0,0 @@
#!/usr/bin/env sh
exec clojure -M:dev:tests "$@"

11
backend/src/app/config.clj

@ -16,6 +16,7 @@
[app.util.time :as dt]
[clojure.core :as c]
[clojure.pprint :as pprint]
[clojure.java.io :as io]
[clojure.spec.alpha :as s]
[cuerdas.core :as str]
[environ.core :refer [env]]))
@ -50,7 +51,7 @@
:storage-backend :fs
:storage-fs-directory "resources/public/assets"
:storage-fs-directory "assets"
:storage-s3-region :eu-central-1
:storage-s3-bucket "penpot-devenv-assets-pre"
@ -246,15 +247,17 @@
{}
env)))
(defn- read-config
[]
(->> (read-env "penpot")
(merge defaults)
(us/conform ::config)))
(def version (v/parse "%version%"))
(def config (atom (read-config)))
(def version (v/parse (or (some-> (io/resource "version.txt")
(slurp)
(str/trim))
"%version%")))
(def config (atom (read-config)))
(def deletion-delay
(dt/duration {:days 7}))

3
docker/devenv/Dockerfile

@ -6,7 +6,7 @@ ARG DEBIAN_FRONTEND=noninteractive
ENV NODE_VERSION=v14.16.0 \
CLOJURE_VERSION=1.10.3.814 \
CLJKONDO_VERSION=2021.03.22 \
BABASHKA_VERSION=0.3.0 \
BABASHKA_VERSION=0.3.1 \
LANG=en_US.UTF-8 \
LC_ALL=en_US.UTF-8
@ -151,6 +151,7 @@ EXPOSE 3449
EXPOSE 6060
EXPOSE 9090
COPY files/nginx.conf /etc/nginx/nginx.conf
COPY files/phantomjs-mock /usr/bin/phantomjs
COPY files/bashrc /root/.bashrc

1
docker/devenv/docker-compose.yaml

@ -27,7 +27,6 @@ services:
volumes:
- "user_data:/home/penpot/"
- "${PWD}:/home/penpot/penpot"
- ./files/nginx.conf:/etc/nginx/nginx.conf
ports:
- 3447:3447

2
docker/devenv/files/nginx.conf

@ -91,7 +91,7 @@ http {
location /internal/assets {
internal;
alias /home/penpot/penpot/backend/resources/public/assets;
alias /home/penpot/penpot/backend/assets;
add_header x-internal-redirect "$upstream_http_x_accel_redirect";
}

2
docker/devenv/files/start-tmux.sh

@ -26,7 +26,9 @@ tmux send-keys -t penpot 'npx shadow-cljs watch main' enter
tmux new-window -t penpot:2 -n 'exporter'
tmux select-window -t penpot:2
tmux send-keys -t penpot 'cd penpot/exporter' enter C-l
tmux send-keys -t penpot 'rm -f target/app.js*' enter C-l
tmux send-keys -t penpot 'npx shadow-cljs watch main' enter
tmux split-window -v
tmux send-keys -t penpot 'cd penpot/exporter' enter C-l
tmux send-keys -t penpot './scripts/wait-and-start.sh' enter

1
exporter/scripts/build.sh → exporter/scripts/build

@ -1,6 +1,5 @@
#!/usr/bin/env bash
source ~/.bashrc
set -ex
yarn install

17
exporter/scripts/wait-and-start.sh

@ -1,16 +1,5 @@
#!/usr/bin/env bash
set -e
wait_file() {
local file="$1"; shift
local wait_seconds="${1:-10}"; shift # 10 seconds as default timeout
until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do sleep 1; done
((++wait_seconds))
}
wait_file "target/app.js" 120 && {
node target/app.js
}
bb -i '(babashka.wait/wait-for-port "localhost" 9630)';
bb -i '(babashka.wait/wait-for-path "target/app.js")';
node target/app.js

17
frontend/scripts/build

@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -ex
CURRENT_VERSION=$1;
CURRENT_HASH=$(git rev-parse --short HEAD);
EXTRA_PARAMS=$SHADOWCLJS_EXTRA_PARAMS;
yarn install || exit 1;
npx gulp clean || exit 1;
npx shadow-cljs release main --config-merge "{:release-version \"${CURRENT_HASH}\"}" $EXTRA_PARAMS || exit 1
npx gulp build || exit 1;
npx gulp dist:clean || exit 1;
npx gulp dist:copy || exit 1;
sed -i -re "s/\%version\%/$CURRENT_VERSION/g" ./target/dist/index.html;

8
frontend/scripts/build-and-run-tests.sh

@ -1,8 +0,0 @@
#!/usr/bin/env bash
source ~/.bashrc
set -ex;
yarn install
clojure -Adev tools.clj build:tests
node ./target/tests/main

21
frontend/scripts/build.sh

@ -1,21 +0,0 @@
#!/usr/bin/env bash
source ~/.bashrc
set -ex
if [ -z "${TAG}" ]; then
export TAG=$(git log -n 1 --pretty=format:%H -- ./);
fi
yarn install
export NODE_ENV=production;
# Clean the output directory
npx gulp clean || exit 1;
npx shadow-cljs release main --config-merge "{:release-version \"${TAG}\"}" $SHADOWCLJS_EXTRA_PARAMS
npx gulp build || exit 1;
npx gulp dist:clean || exit 1;
npx gulp dist:copy || exit 1;

9
frontend/src/app/main/ui/workspace/viewport/hooks.cljs

@ -96,10 +96,11 @@
(mf/deps page-id)
(fn [point]
(let [rect (gsh/center->rect point 8 8)]
(uw/ask! {:cmd :selection/query
:page-id page-id
:rect rect
:include-frames? true}))))
(uw/ask-buffered!
{:cmd :selection/query
:page-id page-id
:rect rect
:include-frames? true}))))
;; We use ref so we don't recreate the stream on a change
transform-ref (mf/use-ref nil)

4
frontend/src/app/main/worker.cljs

@ -25,3 +25,7 @@
(defn ask!
[message]
(uw/ask! instance message))
(defn ask-buffered!
[message]
(uw/ask-buffered! instance message))

73
frontend/src/app/util/worker.cljs

@ -17,39 +17,58 @@
(declare handle-response)
(defrecord Worker [instance stream])
(defn ask!
[w message]
(let [sender-id (uuid/next)
data (t/encode {:payload message :sender-id sender-id})
instance (:instance w)]
(defn- send-message! [worker {sender-id :sender-id :as message}]
(let [data (t/encode message)
instance (:instance worker)]
(.postMessage instance data)
(->> (:stream w)
(->> (:stream worker)
(rx/filter #(= (:reply-to %) sender-id))
(rx/map handle-response)
(rx/first))))
(rx/take 1)
(rx/map handle-response))))
(defn ask!
[worker message]
(send-message!
worker
{:sender-id (uuid/next)
:payload message}))
(defn ask-buffered!
[worker message]
(send-message!
worker
{:sender-id (uuid/next)
:payload message
:buffer? true}))
(defn init
"Return a initialized webworker instance."
[path on-error]
(let [ins (js/Worker. path)
bus (rx/subject)
wrk (Worker. ins bus)]
(.addEventListener ins "message"
(fn [event]
(let [data (.-data event)
data (t/decode data)]
(if (:error data)
(on-error (:error data))
(rx/push! bus data)))))
(.addEventListener ins "error"
(fn [error]
(on-error wrk (.-data error))))
wrk))
(let [instance (js/Worker. path)
bus (rx/subject)
worker (Worker. instance bus)
handle-message
(fn [event]
(let [data (.-data event)
data (t/decode data)]
(if (:error data)
(on-error (:error data))
(rx/push! bus data))))
handle-error
(fn [error]
(on-error worker (.-data error)))]
(.addEventListener instance "message" handle-message)
(.addEventListener instance "error" handle-error)
worker))
(defn- handle-response
[{:keys [payload error] :as response}]
(if-let [{:keys [data message]} error]
(throw (ex-info message data))
payload))
[{:keys [payload error dropped] :as response}]
(when-not dropped
(if-let [{:keys [data message]} error]
(throw (ex-info message data))
payload)))

76
frontend/src/app/worker.cljs

@ -28,14 +28,23 @@
;; --- Messages Handling
(s/def ::cmd keyword?)
(s/def ::payload
(s/keys :req-un [::cmd]))
(s/def ::sender-id uuid?)
(s/def ::buffer? boolean?)
(s/def ::message
(s/keys :req-un [::payload ::sender-id]))
(s/keys
:req-opt [::buffer?]
:req-un [::payload ::sender-id]))
(def buffer (rx/subject))
(defn- handle-message
"Process the message and returns to the client"
[{:keys [sender-id payload] :as message}]
(us/assert ::message message)
(try
@ -68,20 +77,83 @@
:message (ex-message e)}}]
(.postMessage js/self (t/encode message))))))
(defn- drop-message
"Sends to the client a notifiction that its messages have been dropped"
[{:keys [sender-id payload] :as message}]
(us/assert ::message message)
(.postMessage js/self (t/encode {:reply-to sender-id
:dropped true})))
(defn subscribe-buffer-messages
"Creates a subscription to process the buffer messages"
[]
(let [empty [{} [] ::clear]]
(->> buffer
;; We want async processing to not block the main loop
(rx/observe-on :async)
;; This scan will store the last message per type in `messages`
;; when a previous message is dropped is stored in `dropped`
;; we also store the last message processed in order to detect
;; posible infinite loops
(rx/scan
(fn [[messages dropped last] message]
(let [cmd (get-in message [:payload :cmd])
;; The previous message is dropped
dropped
(cond-> dropped
(contains? messages cmd)
(conj (get messages cmd)))
;; This is the new "head" for its type
messages
(assoc messages cmd message)]
;; When a "clear" message is detected we empty the buffer
(if (= message ::clear)
empty
[messages dropped message])))
empty)
;; 1ms debounce, after 1ms without messages will process the buffer
(rx/debounce 1)
(rx/subs (fn [[messages dropped last]]
;; Send back the dropped messages replies
(doseq [msg dropped]
(drop-message msg))
;; Process the message
(doseq [msg (vals messages)]
(handle-message msg))
;; After process the buffer we send a clear
(when-not (= last ::clear)
(rx/push! buffer ::clear)))))))
(defonce process-message-sub (subscribe-buffer-messages))
(defn- on-message
[event]
(when (nil? (.-source event))
(let [message (.-data event)
message (t/decode message)]
(handle-message message))))
(if (:buffer? message)
(rx/push! buffer message)
(handle-message message)))))
(.addEventListener js/self "message" on-message)
(defn ^:dev/before-load stop []
(rx/-dispose process-message-sub)
(.removeEventListener js/self "message" on-message))
(defn ^:dev/after-load start []
[]
(set! process-message-sub (subscribe-buffer-messages))
(.addEventListener js/self "message" on-message))

38
manage.sh

@ -71,6 +71,9 @@ function run-devenv {
}
function build {
echo ">> build start: $1"
local version=$(print-current-version);
pull-devenv-if-not-exists;
docker volume create ${DEVENV_PNAME}_user_data;
docker run -t --rm \
@ -79,10 +82,28 @@ function build {
-e EXTERNAL_UID=$CURRENT_USER_ID \
-e SHADOWCLJS_EXTRA_PARAMS=$SHADOWCLJS_EXTRA_PARAMS \
-w /home/penpot/penpot/$1 \
$DEVENV_IMGNAME:latest sudo -EH -u penpot ./scripts/build.sh
$DEVENV_IMGNAME:latest sudo -EH -u penpot ./scripts/build $version
echo ">> build end: $1"
}
function put-license-file {
local target=$1;
tee -a $target/LICENSE >> /dev/null <<EOF
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
This Source Code Form is "Incompatible With Secondary Licenses", as
defined by the Mozilla Public License, v. 2.0.
Copyright (c) UXBOX Labs SL
EOF
}
function build-app-bundle {
echo ">> bundle app start";
local version=$(print-current-version);
local bundle_dir="./bundle-app";
@ -91,25 +112,28 @@ function build-app-bundle {
rm -rf $bundle_dir
mkdir -p $bundle_dir;
cp -r ./frontend/target/dist $bundle_dir/frontend;
cp -r ./backend/target/dist $bundle_dir/backend;
mv ./frontend/target/dist $bundle_dir/frontend;
mv ./backend/target/dist $bundle_dir/backend;
echo $version > $bundle_dir/version.txt
sed -i -re "s/\%version\%/$version/g" $bundle_dir/frontend/index.html;
sed -i -re "s/\%version\%/$version/g" $bundle_dir/backend/main/app/config.clj;
put-license-file $bundle_dir;
echo ">> bundle app end";
}
function build-exporter-bundle {
echo ">> bundle exporter start";
local version=$(print-current-version);
local bundle_dir="./bundle-exporter";
build "exporter";
rm -rf $bundle_dir;
cp -r ./exporter/target $bundle_dir;
mv ./exporter/target $bundle_dir;
echo $version > $bundle_dir/version.txt
put-license-file $bundle_dir;
echo ">> bundle exporter end";
}
function usage {

Loading…
Cancel
Save