edit

README.md

@@ -245,4 +245,13 @@ ds num web-nginx 10:将nginx集群的数量增加到10
 6、dever create 在当前目录下创建一个dever项目
 7、dever all 更新当前所有组件
 
-</pre>
+</pre>
+
+
+<pre>
+2018-9-10更新：
+1、增加文件同步工具包：rsync、btsync、lsyncd、nfs等，可以根据需要来使用。
+2、增加overlay网络的跨主机支持，增加docker run可以直接使用swarm网络，方便docker单机与集群的联通。添加overflow网络时，建议加上overlay_前缀
+3、daemon进程中增加文件同步机制，会将share/lib中的文件同步到从机，daemon中的git容器会自动更新代码到share/lib/php/web中
+
+</pre>

container/conf/web/nginx/conf.d/site.conf

@@ -24,6 +24,9 @@ server {
     if ( $request_uri ~* /system/wechat ) {
             rewrite ^/system/wechat /system/index.php?m=payment&a=wechat_notify last;
     }
+    if ( $request_uri ~* /grow/pay/notify ) {
+            rewrite ^/grow/pay/notify/([0-9+]) /grow/pay/index.php?l=api.notify&account_id=$1 last;
+    }
     location / {
                 # First attempt to serve request as file, then
                 # as directory, then fall back to displaying a 404.
@@ -38,7 +41,7 @@ server {
 
         # With php-cgi alone:
         #fastcgi_pass web-php7:9000;
-        fastcgi_pass onepage-php5:9000;
+        fastcgi_pass web-php5:9000;
         # With php-fpm:
         #fastcgi_pass unix:/var/run/php7-fpm.sock;
         fastcgi_index index.php;

container/conf/web/php5/conf.d/sockets.ini

@@ -1,2 +1 @@
 extension=sockets.so
-extension=event.so

container/conf/web/php5/php.ini

@@ -2033,3 +2033,4 @@ ldap.max_links = -1
 ; Local Variables:
 ; tab-width: 4
 ; End:
+zend_extension=ioncube_loader_lin_5.6.so

install

@@ -34,5 +34,7 @@ ln -sf $basepath/src/dpc.py /usr/bin/dpc && chmod +x /usr/bin/dpc
 ln -sf $basepath/src/dever.py /usr/bin/dever && chmod +x /usr/bin/dever
 ln -sf $basepath/src/dgit.py /usr/bin/dgit && chmod +x /usr/bin/dgit
 
+python $basepath/src/daemon.py &
+
 docker version
 echo 'install success!'

src/docker/build/dev/php/v5/Dockerfile

@@ -1,10 +1,10 @@
-FROM docker.dever.cc/os/alpine/v3.4:latest
+FROM docker.dever.cc/os/alpine:latest
 
 MAINTAINER Rabin "https://github.com/shemic"
 
 ENV COMPOSER_HOME=/share/lib/php
 
-RUN apk add --no-cache --update curl php5-fpm php5-pear php5-mysql php5-pgsql php5-pdo_mysql php5-pdo_pgsql php5-gd php5-curl php5-mcrypt php5-json php5-zlib php5-xml php5-iconv php5-zip php5-phar php5-openssl php5-dom php5-sockets php5-pcntl && curl -sS https://getcomposer.org/installer | php && mv composer.phar /usr/bin/composer && apk del curl && composer config -g repo.packagist composer https://packagist.phpcomposer.com
+RUN apk add --no-cache --update curl php5-fpm php5-pear php5-mysql php5-pgsql php5-pdo_mysql php5-pdo_pgsql php5-gd php5-curl php5-mcrypt php5-json php5-zlib php5-xml php5-iconv php5-zip php5-phar php5-openssl php5-dom php5-sockets php5-pcntl && ln /usr/bin/php5 /usr/bin/php && curl -sS https://getcomposer.org/installer | php5 && mv composer.phar /usr/bin/composer && apk del curl && composer config -g repo.packagist composer https://packagist.phpcomposer.com
 
 COPY php.sh /entrypoint/php.sh
 COPY install.sh /install.sh

src/docker/build/dev/php/v5/install.sh

@@ -10,15 +10,15 @@ curl -O http://pecl.php.net/get/$1.tgz
 tar -xzvf $1.tgz
 rm -rf $1.tgz
 cd $1
-phpize
+phpize5
 if [ -n "$4" ];then
 	config=$4
-	./configure --with-php-config=/usr/bin/php-config ${config//,/" "}
+	./configure --with-php-config=/usr/bin/php-config5 ${config//,/" "}
 else
-	./configure --with-php-config=/usr/bin/php-config
+	./configure --with-php-config=/usr/bin/php-config5
 fi
 make
 make install
 echo extension=$2.so > /etc/php5/conf.d/$2.ini
-killall -9 php-fpm && php-fpm &
+killall -9 php-fpm5 && php-fpm5 &
 #apk del $lib

src/docker/build/dev/php/v5/php.sh

@@ -1,19 +1,22 @@
 #!/usr/bin/env sh
 set -e
-PHP="php-fpm"
+PHP="php-fpm5"
 start_php()
 {
 	# 使用exec 将替换主进程，信号检测将失效，无法执行end_php
 	#exec php-fpm
 	process_start $PHP
+	process_start crond
 }
 
 stop_php()
 {
 	process_stop $PHP
+	process_stop crond
 }
 
 monit_php()
 {
 	process_monit $PHP
+	process_monit crond
 }

src/docker/build/dev/php/v7/php.sh

@@ -6,14 +6,17 @@ start_php()
 	# 使用exec 将替换主进程，信号检测将失效，无法执行end_php
 	#exec php-fpm
 	process_start $PHP
+	process_start crond
 }
 
 stop_php()
 {
 	process_stop $PHP
+	process_stop crond
 }
 
 monit_php()
 {
 	process_monit $PHP
+	process_monit crond
 }

src/docker/build/file/btsync/Dockerfile

@@ -0,0 +1,3 @@
+FROM ctlc/btsync:latest
+
+MAINTAINER Rabin "https://github.com/shemic"

src/docker/build/file/lsyncd/Dockerfile

@@ -0,0 +1,11 @@
+FROM docker.dever.cc/os/alpine:latest
+
+MAINTAINER Rabin "https://github.com/shemic"
+
+RUN apk --no-cache add rsync lua lua-dev git gcc g++ libgc++ make cmake && \
+	git clone https://github.com/axkibe/lsyncd.git && \
+	cd lsyncd && \
+	cmake . && \
+	make && make install && \
+	apk del git gcc g++ libgc++ make cmake && \
+    rm -rf /tmp/* /var/cache/apk/*

src/docker/build/file/rsync/Dockerfile

@@ -0,0 +1,9 @@
+FROM docker.dever.cc/os/alpine:latest
+
+MAINTAINER Rabin "https://github.com/shemic"
+
+RUN apk update && apk add --no-cache rsync
+
+COPY rsync.sh /entrypoint/rsync.sh
+
+CMD ["rsync"]

src/docker/build/file/rsync/rsync.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env sh
+set -e
+start_rsync()
+{
+	process_start crond
+}
+
+stop_rsync()
+{
+	process_stop crond
+}
+
+monit_rsync()
+{
+	process_monit crond
+}

src/docker/build/office/convert/Dockerfile

@@ -6,13 +6,17 @@ ENV DEMETER_HOME=/usr/local/convert
 ENV DEMETER_CONF=env
 ENV PATH=$PATH:$DEMETER_HOME
 
-RUN apk add --no-cache --update git gcc curl gcc g++ python-dev py-pip imagemagick imagemagick-dev && \
+RUN apk add --no-cache --update git curl gcc g++ python-dev py-pip openssl-dev libffi-dev imagemagick imagemagick-dev && \
 	pip install --upgrade pip && \
 	pip install gevent==1.2 && \
 	pip install redis && \
 	pip install wand && \
 	pip install -U git+http://git.dever.cc:3000/python/demeter.git && \
 	git clone http://git.dever.cc:3000/python/convert.git $DEMETER_HOME && \
+	curl -O https://fossies.org/linux/misc/poppler-data-0.4.9.tar.gz && \
+	tar -zxvf poppler-data-0.4.9.tar.gz && \
+	cd poppler-data-0.4.9 && \
+	make install datadir=/usr/share && \
 	#chmod -R +x $DEMETER_HOME/*.py && \
 	apk del curl gcc g++ python-dev
 

src/docker/build/os/alpine/daemon/Dockerfile

@@ -0,0 +1,9 @@
+FROM docker.dever.cc/os/alpine:latest
+
+MAINTAINER Rabin "https://github.com/shemic"
+
+COPY daemon.sh /entrypoint/daemon.sh
+COPY daemon.py /usr/bin/daemon
+RUN apk add --no-cache --update python git && chmod +x /usr/bin/daemon
+
+CMD ["daemon"]

src/docker/build/os/alpine/daemon/daemon.sh

@@ -0,0 +1,17 @@
+#!/usr/bin/env sh
+set -e
+commands="daemon"
+start_daemon()
+{
+	process_start $commands
+}
+
+stop_daemon()
+{
+	process_stop $commands
+}
+
+monit_daemon()
+{
+	process_monit $commands
+}

src/docker/build/os/alpine/glibc/Dockerfile

@@ -2,9 +2,13 @@ FROM docker.dever.cc/os/alpine:latest
 
 MAINTAINER Rabin "https://github.com/shemic"
 
-ENV GLIB_VERSION 2.25-r0
-RUN apk add --no-cache --update curl ca-certificates && \
-	curl -Ls https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIB_VERSION}/glibc-${GLIB_VERSION}.apk > /tmp/glibc-${GLIB_VERSION}.apk && \
-    apk add --allow-untrusted /tmp/glibc-${GLIB_VERSION}.apk && \
-    apk del curl ca-certificates && \
+#curl -Ls
+
+ENV GLIB_VERSION 2.28-r0
+RUN apk --no-cache add ca-certificates wget && \
+	wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \
+	wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIB_VERSION}/glibc-${GLIB_VERSION}.apk && \
+	wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIB_VERSION}/glibc-i18n-${GLIB_VERSION}.apk && \
+	apk add glibc-${GLIB_VERSION}.apk glibc-i18n-${GLIB_VERSION}.apk && \
+	apk del wget ca-certificates && \
     rm -rf /tmp/* /var/cache/apk/*

src/docker/build/web/openresty/Dockerfile

@@ -9,6 +9,7 @@ ENV PATH=$PATH:$OR_HOME/bin:$OR_HOME/nginx/sbin
 RUN apk add --no-cache --update curl gcc make perl libpcre32 pcre libgcc libressl-dev openssl musl-dev pcre-dev zlib-dev && \
 	curl -O https://openresty.org/download/openresty-${OR_VERSION}.tar.gz && \
 	tar -zxvf openresty-${OR_VERSION}.tar.gz && \
+	rm -rf openresty-${OR_VERSION}.tar.gz && \
 	cd openresty-${OR_VERSION} && \
 	./configure && \
 	make && make install && \

src/docker/conf/chengmao.conf

@@ -37,4 +37,12 @@ volumes = {container}web/jiazhi:/root/input,{container}web/jiazhi/doc:/root/outp
 call = apidoc
 #使用dm call iot-apidoc input=demo&out=output来运行，然后可以在宿主机里使用apidoc命令
 param = -i /root/input/doc -o /root/output -c /root/config
+alias = apidoc
+
+[leek_apidoc]
+image = apidoc
+volumes = {container}web/leek:/root/input,{container}web/leek/doc:/root/output,{container}web/leek/doc:/root/config
+call = apidoc
+#使用dm call iot-apidoc input=demo&out=output来运行，然后可以在宿主机里使用apidoc命令
+param = -i /root/input/doc -o /root/output -c /root/config
 alias = apidoc

src/docker/conf/daemon.conf

@@ -1,9 +1,14 @@
 [base]
 path = {base}
-default = master
+default = master,manager
 network = overlay_dm
 subnet = 10.0.0.0/255
 
+;别名注册，使用 dm run daemon-run
+[alias]
+run = master,manager,operater
+add = client,worker
+
 ;服务注册
 [master]
 image = consul
@@ -21,20 +26,28 @@ volumes = {container}conf/service/consul:/root/consul/config
 command = consul -data-dir=/root/consul/data -client=0.0.0.0 -join={parent} -dc=daemon-master
 alias = consul->consul_client
 
+;任务管理
+[crond]
+image = alpine_daemon
+command = daemon
+
 ;文件共享与管理者
 [manager]
-image = nfs
-super = true
+image = btsync
+volumes = {container}share/lib:/data
+;获取超级管理权限
+;super = true
 
-;文件发现
+;文件发现 dm run daemon-worker input=A4IDCT5EADNLSWEZ3V5C3ONPGBBGEUDJD
 [worker]
-image = nfs
-super = true
+image = btsync
+command = {$input:A4IDCT5EADNLSWEZ3V5C3ONPGBBGEUDJD}
+volumes = {container}share/lib:/data
 
 ;操作者，图形界面后台，可以直接访问daemon-manager
 [operater]
 image = admin
 port = 8087:8011,8088:8012
 
-;master、manager、operater为主机开启
+;master、manager、operater、crond为主机开启
 ;client、worker为其他机器开启

src/docker/conf/tool.conf

@@ -10,7 +10,7 @@ command = zeppelin
 [apidoc]
 volumes = {container}src/apidoc/input:/root/input,{container}web/apidoc:/root/output,{container}src/apidoc/config:/root/config
 call = apidoc
-#使用dever call tool apidoc input=demo&out=output来运行，然后可以在宿主机里使用apidoc命令
+#使用dever call tool-apidoc input=demo&out=output来运行，然后可以在宿主机里使用apidoc命令
 param = -i /root/input/{$input:demo}/ -o /root/{$out:output} -c /root/config
 alias = apidoc
 
@@ -23,5 +23,18 @@ port = 9030:80,6800:6800
 ;dweb
 volumes = {container}share/data/pan:/var/www/html/system/data/default_home_folder
 
-[grafana]
-port = 3001:3000
+[nfs]
+port = 3001:3000
+
+;文件共享与管理者
+[file_manager]
+image = btsync
+volumes = {container}share/sync_server:/data
+;获取超级管理权限
+;super = true
+
+;文件发现 dm run daemon-worker input=A4IDCT5EADNLSWEZ3V5C3ONPGBBGEUDJD
+[file_worker]
+image = btsync
+command = {$input:A4IDCT5EADNLSWEZ3V5C3ONPGBBGEUDJD}
+volumes = {container}share/sync_client:/data

src/docker/conf/web.conf

@@ -3,7 +3,7 @@ path = {base}
 default = mysql,php7,nginx
 
 [mysql]
-port = 3309:3306
+#port = 3309:3306
 volumes = {container}conf/db/mysql:/etc/mysql,/mysql/{name}/data:/var/lib/mysql
 environment = MYSQL_ROOT_PASSWORD=123456
 alias = mysql
@@ -11,7 +11,7 @@ alias = mysql
 [mariadb]
 #使用mariadb来代替mysql
 image = mariadb
-port = 3310:3306
+#port = 3310:3306
 volumes = {container}conf/db/mariadb:/etc/mysql,/mysql/{name}/data:/var/lib/mysql
 environment = MYSQL_ROOT_PASSWORD=123456
 alias = mysql
@@ -33,7 +33,7 @@ alias = php,composer,pecl,apk,/install.sh->phpInstall
 volumes = {container}web:/www,{container}conf/web/php7:/etc/php7
 alias = php,composer,pecl,apk,/install.sh->phpInstall
 
-[nginx]
+[#nginx]
 #num = 3
 # num和cluster都可以设置有多少个nginx进程，不同的是,cluster仅对ds有效
 cluster = 3
@@ -41,9 +41,9 @@ port = 80:80,443:443
 volumes = {container}web:/www,{container}conf/web/nginx:/etc/nginx,{container}logs/nginx/{name}/logs:/var/log/nginx
 alias = nginx
 
-[#nginx]
+[nginx]
 #使用openresty来代替nginx，可以写lua脚本了
 image = openresty
 port = 80:80,443:443
 volumes = {container}web:/www,{container}conf/web/nginx:/usr/local/openresty/nginx/conf,{container}logs/nginx/openresty_{name}/logs:/var/log/nginx
-command = nginx
+command = nginx

src/docker/core.conf

@@ -9,6 +9,7 @@ aliyun				= registry.cn-hangzhou.aliyuncs.com/shemic
 alpine				= os/alpine
 alpine3.4			= os/alpine/v3.4
 alpine_glibc		= os/alpine/glibc
+alpine_daemon		= os/alpine/daemon
 ubuntu				= os/ubuntu
 debian				= os/debian
 
@@ -87,7 +88,7 @@ dlib				= learning/dlib
 gogs				= base/gogs
 registry			= base/registry
 redmine				= base/redmine
-swarm               = base/swarm
+swarm				= base/swarm
 
 ;service
 consul				= service/consul
@@ -103,9 +104,9 @@ project_buy			= project/buy
 project_onepage		= project/onepage
 
 ;spider
-xvfb_firefox        = spider/xvfb/firefox
-xvfb_chrome         = spider/xvfb/chrome
-phantomjs           = spider/phantomjs
+xvfb_firefox		= spider/xvfb/firefox
+xvfb_chrome			= spider/xvfb/chrome
+phantomjs			= spider/phantomjs
 
 ;office
 libreoffice			= office/libreoffice
@@ -116,4 +117,7 @@ convert				= office/convert
 lvs					= balance/lvs
 
 ;file
-nfs                 = file/nfs
+nfs					= file/nfs
+btsync				= file/btsync
+rsync				= file/rsync
+lsyncd				= file/lsyncd

src/shell/git/hook

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+set -e
+#unset $(git rev-parse --local-env-vars)
+unset GIT_DIR
+dir="{path}"
+logfile="{path}.git/hooks_log"
+cd $dir
+git pull origin master
+currTime="{path}: git pull at "$(date "+%Y-%m-%d %H:%M:%S")
+echo $currTime >> $logfile

src/tool/cluster.py

@@ -217,48 +217,75 @@ class Cluster_Action(Docker_Action):
 			Env.cluster(ckey)
 		return (ip, ckey)
 
+	@classmethod
+	def put(self, key = '', value = '', ip = '', p = True):
+		if not ip:
+			ip = Core.ip()
+		ip = Core.ip()
+		if not key:
+			key = Args.name
+		if not value:
+			value = Args.param
+		url = 'http://' + ip + ':8500/v1/kv/' + key
+		Core.popen('curl -X PUT -d "'+value+'" ' + url, bg=True)
+		if p == True:
+			print True
+		return True
+
+	@classmethod
+	def get(self, key = '', ip = '', p = True):
+		if not ip:
+			ip = Core.ip()
+		if not key:
+			key = Args.name
+		import json
+		import base64
+		url = 'http://' + ip + ':8500/v1/kv/' + key
+		value = Core.curl(url)
+		if not value:
+			print False
+			return False
+		value = json.loads(value)
+		value = base64.b64decode(value[0]['Value'])
+		if p == True:
+			print value
+		return value
+
 	@classmethod
 	def init(self, **param):
 		(ip, ckey) = self.setting()
 		token = Swarm.init(ip)
-		
+		print 'init cluster ...'
 		if token and '--token' in token:
 			Core.popen('dm pull consul')
 			Core.popen('ds run daemon-master')
 
 			token = token.split('docker swarm join --token ')
-			token = token[1].split("\r\n")
+			token = token[1].split("\n")
 			token = token[0].replace(' ', ':')
 			data = token.split(':')
 			ip = data[1]
 
-			Core.popen('consul kv put ' + ckey + ' ' + token)
-		print 'init cluster('+ip+'):yes'
+			self.put(ckey, token, p = False)
+			#Core.popen('consul kv put ' + ckey + ' ' + token)
+		print 'init cluster success! please remember the ip address:'+ip+''
 
 	@classmethod
 	def join(self, **param):
 		(ip, ckey) = self.setting()
-		import json
-		import base64
-		url = 'http://' + ip + ':8500/v1/kv/' + ckey
-		value = Core.curl(url)
-		print value
-		value = json.loads(value)
-		if not value:
-			print 'join cluster:'+url+' error'
-		else:
-			value = base64.b64decode(value[0]['Value'])
-			config = value.split(':')
-			print config
+		value = self.get(ckey, ip, False)
+		config = value.split(':')
+		print config
 
-			'''
-			Core.popen('dm pull consul')
-			Swarm.join(config[0])
+		'''
+		Core.popen('dm pull consul')
+		Swarm.join(config[0])
 
-			Core.popen('dm run daemon-client')
+		Core.popen('dm run daemon-client')
 
-			print 'join cluster:yes'
-			'''
+		print 'join cluster:yes'
+		'''
+			
 
 class Swarm(object):
 	@staticmethod

src/tool/docker.py

@@ -37,7 +37,9 @@ class Docker(object):
 			self.handle(method, self.conf['config'][Args.index], Args.index, Args.action)
 		else:
 			server = self.conf['server']
-			if 'default' in self.conf['base']:
+			if 'alias' in self.conf['config'] and Args.index in self.conf['config']['alias']:
+				server = self.conf['config']['alias'][Args.index].split(',')
+			elif 'default' in self.conf['base']:
 				server = self.conf['base']['default'].split(',')
 			for item in server:
 				if self.check(Args.index, item) == True:

src/tool/php.py

@@ -15,6 +15,7 @@ class Php(object):
 
 class Php_Action(object):
 	package = {
+		#http://pecl.php.net
 		# 名称-版本号,so名,依赖包,configure参数
 		'libevent' : ['event-2.3.0', 'event', 'curl-dev,libevent-dev', '']
 		,'swoole' : ['swoole-2.0.10', 'swoole', 'libevent-dev,libaio-dev,libmnl-dev', '']