发布于2021的文章

Oct 07

哦,原来视频流式传输需要这么优化: Optimizing MP4 Video for Fast Streaming,那么就撸个批处理干,支持 mp4/mkv/mov/m4v 格式的优化。

#!/bin/bash

[ -n "$1" ] && {
    src_folder="$1"
} || {
    echo "input you source folder location."
    exit
}

src_empty=$(ls -A "$src_folder")

[ -n "$src_empty" ] && {
    for file_a in ${src_folder}/*
    do
        in_full_filename=$(basename "$file_a")
        in_filename="${filename%.*}"
        in_extension="${file_a##*.}"
        out_full_filename="${in_filename}_.${in_extension}"

        echo "src=$in_full_filename ndst=$out_full_filename"

        case $in_extension in
            mp4|mov|m4v)
                ffmpeg -i "${src_folder}/${in_full_filename}" -movflags faststart -acodec copy -vcodec copy -copyts "${src_folder}/$out_full_filename"
                ;;
            mkv)
                ffmpeg -i "${src_folder}/${in_full_filename}" -c copy -reserve_index_space 100k -copyts "${src_folder}/$out_full_filename"
                ;;
            *)
                echo "${in_extension} file not support faster start."
                ;;
        esac

        [ $? -eq 0 ] && {
            rm -f "${src_folder}/${in_full_filename}"
            mv -f "${src_folder}/$out_full_filename" "${src_folder}/${in_full_filename}"
            echo "-----------${in_filename} now faster web loading------------"
        } || {
            echo "-----------${in_filename} faster process failed------------"
        }
    done
} || {
    echo "Source folder is empty."
}

Windows 版本,丢到要处理的视频目录下:

for /r %%F in (*.mp4) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -acodec copy -vcodec copy -copyts "%%~dpnF"_.mp4
    if not errorlevel 1 if exist "%%~dpnF.mp4" (
		del /q "%%F"
		move "%%~dpnF"_.mp4 "%%F"
	)
)

for /r %%F in (*.mov) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -acodec copy -vcodec copy -copyts "%%~dpnF"_.mov
    if not errorlevel 1 if exist "%%~dpnF.mov" (
		del /q "%%F"
		move "%%~dpnF"_.mov "%%F"
	)
)

for /r %%F in (*.mkv) do (
    ffmpeg.exe  -i "%%F" -c copy -reserve_index_space 100k -copyts "%%~dpnF"_.mkv
    if not errorlevel 1 if exist "%%~dpnF.mkv" (
		del /q "%%F"
		move "%%~dpnF"_.mkv "%%F"
	)
)

for /r %%F in (*.wmv) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -copyts "%%~dpnF".mp4
    if not errorlevel 1 if exist "%%~dpnF.wmv" (
		del /q "%%F"
	)
)

for /r %%F in (*.mpg) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -copyts "%%~dpnF".mp4
    if not errorlevel 1 if exist "%%~dpnF.mpg" (
		del /q "%%F"
	)
)

for /r %%F in (*.flv) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -copyts "%%~dpnF".mp4
    if not errorlevel 1 if exist "%%~dpnF.flv" (
		del /q "%%F"
	)
)

for /r %%F in (*.rmvb) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -copyts "%%~dpnF".mp4
    if not errorlevel 1 if exist "%%~dpnF.rmvb" (
		del /q "%%F"
	)
)

for /r %%F in (*.rm) do (
    ffmpeg.exe  -i "%%F" -movflags faststart -copyts "%%~dpnF".mp4
    if not errorlevel 1 if exist "%%~dpnF.rm" (
		del /q "%%F"
	)
)

Oct 06

为了防止滥用,先生成认证文件:

#生成密码文件,添加用户名
sudo sh -c "echo -n 'sammy:' >> /usr/local/nginx/conf/.htpasswd"
#为此用户设置密码
sudo sh -c "openssl passwd -apr1 >> /usr/local/nginx/conf/.htpasswd"

1. 通过境外 VPS 反代:

nginx 关键代码:

location /
{
#	开启认证防止滥用
	auth_basic "Once A Thief";
	auth_basic_user_file /usr/local/nginx/conf/.htpasswd;

	proxy_pass http://www.t66y.com;
	proxy_set_header Host www.t66y.com;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	proxy_set_header REMOTE-HOST $remote_addr;

	add_header X-Cache $upstream_cache_status;
	proxy_set_header Accept-Encoding "";
	proxy_ssl_name www.t66y.com;
	proxy_ssl_server_name on;
	sub_filter "www.t66y.com" "usite.domain.com";
	sub_filter_once off;
	expires 12h;
}

location ~ .*.(php|jsp|cgi|asp|aspx|flv|swf|xml)?$
{
	auth_basic "Once A Thief";
	auth_basic_user_file /usr/local/nginx/conf/.htpasswd;
	proxy_pass http://www.t66y.com;
	proxy_set_header Host www.t66y.com;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	proxy_set_header REMOTE-HOST $remote_addr;
	proxy_set_header Accept-Encoding "";
	sub_filter "www.t66y.com" "usite.domain.com";
	sub_filter_once off;
}

location ~ .*.(html|htm|png|gif|jpeg|jpg|bmp|js|css)?$
{
	auth_basic "Once A Thief";
	auth_basic_user_file /usr/local/nginx/conf/.htpasswd;
	proxy_pass http://www.t66y.com;
	proxy_set_header Host www.t66y.com;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	proxy_set_header REMOTE-HOST $remote_addr;
	proxy_set_header Accept-Encoding "";
	sub_filter "www.t66y.com" "usite.domain.com";
	sub_filter_once off;
	expires 24h;
}

2. 通过境内 VPS 反代:

nginx 关键代码:

location /
{
#	turn on auth for this location
	auth_basic "Once A Thief";
	auth_basic_user_file /usr/local/nginx/conf/.htpasswd;

	proxy_pass https://127.0.0.1:1024;
	proxy_set_header Host www.t66y.com;
	proxy_set_header X-Real-IP $remote_addr;
	proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
	proxy_set_header REMOTE-HOST $remote_addr;

	add_header X-Cache $upstream_cache_status;
	proxy_set_header Accept-Encoding "";
	proxy_ssl_name www.t66y.com;
	proxy_ssl_server_name on;
	sub_filter "www.t66y.com" "usite.domain.com";
	sub_filter_once off;
	expires 12h;
}

需要准备一个可以出去的 socks5 代理,v2ray,ss,trojan 随便你用什么,假设监听在本地的 1080 端口;

安装 socat:

apt install socat

测试时可用命令行:

socat -d -d TCP4-LISTEN:1024,bind=127.0.0.1,reuseaddr,fork PROXY:127.0.0.1:t66y.com:443,proxyport=1080

正式工作的 systemd service 文件:

[Unit]
Description=socat
After=network.target

[Service]
Restart=on-failure
RestartSec=5s
ExecStart=/usr/bin/socat TCP4-LISTEN:1024,bind=127.0.0.1,reuseaddr,fork PROXY:127.0.0.1:t66y.com:443,proxyport=1080

[Install]
WantedBy=multi-user.target

参考:

https://stackoverflow.com/questions/46803431/nginx-proxy-pass-over-https-proxy
https://gist.github.com/miyouzi/3e3d57cde402b829aeb1d865b14eaa1a


Oct 06

详情见 MeTube 主页,我这里流水账一下:

#安装、升级 npm / n
apt install npm
npm -g install n
n lts

cd metube/ui

# 安装 Angular,构建 UI
npm install
node_modules/.bin/ng build

# 安装 Python 依赖
cd ..
pip3 install pipenv
pipenv install
python3 -m pip install aiohttp
pipenv install aiohttp

#安装 ffmpeg,合并音频视频,Youtube 新视频基本都需要合并
apt install ffmpeg

# 自定义环境文件测试运行
export DOWNLOAD_DIR=/home/wwwroot/metube
export URL_PREFIX=/metube
pipenv run python3 app/main.py

正常会输出:

INFO:ytdl:waiting for item to download
======== Running on http://0.0.0.0:8081 ========
(Press CTRL+C to quit)

可以浏览器连接: http://VPS-IP:8081,测试一下下载,没问题往下走。

配置 Nginx 转发:

#主转发代码
        location /metube/ {
            proxy_pass http://127.0.0.1:8081;
            proxy_redirect off;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header Host $host;
        }
#没有问题这一块可以不要
        location ~* .(css|js)$ {
            proxy_pass http://127.0.0.1:8081;
            proxy_redirect off;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header Host $host;
        }

添加到开机自动运行:

systemctl edit metube --full --force

粘贴如下代码,metube 源码路径,下载路径,反代目录,改成自己的:

[Unit]
Description=Metube Web Service
After=network.target

[Service]
Environment=DOWNLOAD_DIR=/home/wwwroot/metube
Environment=URL_PREFIX=/metube
Restart=always
Type=simple
WorkingDirectory=/root/src/metube
ExecStart=/usr/local/bin/pipenv run python3 /root/src/metube/app/main.py

[Install]
WantedBy=multi-user.target

然后执行:

systemctl enable --now metube

metube 就启动了,并且下次开机也会自己启动。

追加:
添加简单的密码认证:

#生成密码文件,添加用户名
sudo sh -c "echo -n 'sammy:' >> /etc/nginx/.htpasswd"
#为此用户设置密码
sudo sh -c "openssl passwd -apr1 >> /etc/nginx/.htpasswd"

然后添加到 metube 反代中:

location /metube/ {
    auth_basic           "Administrator’s Area";
    auth_basic_user_file /etc/nginx/.htpasswd";
    ...
}

Oct 06

这里试过反代 ifile share,cloudreve,metube 等都出现这个问题,主页面可以显示,表示反代是没有问题的,但是所有 css,js 文件都是 404,界面显示不完全,很奇怪,你说都是静态文件,但 favicon.ico 那些没问题。

原本反代代码,css/js 404 出错:

        location / {
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP       $remote_addr;
            proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_redirect off;
            proxy_pass http://127.0.0.1:808;
            add_header X-Cache $upstream_cache_status;
            add_header Cache-Control no-cache;
        }

添加以下代码,则正常:

        location ~* \.(gif|png|jpg|css|js|woff|woff2)$ {
            proxy_set_header Host $host;
            proxy_set_header X-Real-IP       $remote_addr;
            proxy_set_header X-Forwarded-For  $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_redirect off;
            proxy_pass http://127.0.0.1:808;
            expires 12h;
        }

这里有个问题,如果反代不是在 / 下,而是子目录下,比如 /metube,按照上面的写法就会影响到 / 下主站,导致主站找不到一些资源文件,我这里就出现修改后主站找不到 favicon.png 的问题,所以我就只添加 css|js 添加子目录匹配:

        location /metube/ {
            proxy_pass http://127.0.0.1:8081;
            proxy_redirect off;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header Host $host;
        }

        location ~ ^/metube/.*\.(css|js)$ {
            proxy_pass http://127.0.0.1:8081;
            proxy_redirect off;
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            proxy_set_header Host $host;
        }

主站 PHP 脚本都是远程调用 CDN 的 css|js,没有本地的,所以没出问题。至少这样暂时能用。 :neutral: :neutral: :neutral:

没找到问题的根本! :twisted:


Oct 03

SNI 分流后获取客户端 IP 一般通过 proxy_protocol 来实现,但分流后的某些程序不能识别 proxy_protocol 怎么办?比如我的 DoH 服务器要 IP 地址,但某木头马并不支持,我开启 proxy_protocol 这马就死了。

我们看代码片段,注意注释。

nginx.conf 主配置文件片段:

stream {
    # 这里就是 SNI 识别,将域名映射成一个配置名
    map $ssl_preread_server_name $backend_name {
        我的域名.坑 web;
        马.我的域名.坑 马;
        # 域名都不匹配情况下的默认值
        default web;
    }
    # 转发到 web 服务器
        upstream web {
        server 127.0.0.1:444;
    }
    # 转发到 马 前置服务器
        upstream 马 {
        server 127.0.0.1:446;
    }
    # 为 马 去除 proxy_protocol
    server {
        #nginx server 443 开启 proxy_protocol 后,分流后的所有服务也必须开启 proxy_protocol,否则会报错
        listen 127.0.0.1:446 proxy_protocol so_keepalive=on;
        proxy_protocol off; #然而,我们在这儿把 proxy_protocol 关闭掉,因为 马 不支持!这是关键
        proxy_connect_timeout 300s;
        proxy_timeout 300s;
        proxy_pass 127.0.0.1:445; #这就是 马 实际吃草的地方
    }

     # 监听 443 并开启 ssl_preread
     server {
         listen 443 reuseport;
         listen [::]:443 reuseport;
         proxy_pass $backend_name;
         ssl_preread on; #开启了分流
         proxy_protocol on; #开启了 proxy_protocol
    }
}

虚拟站点配置文件代码块,大致如下:

server
{
     #nginx server 443 开启 proxy_protocol 后,分流后的所有服务也必须开启 proxy_protocol,否则会报错
    listen 127.0.0.1:444 ssl http2 reuseport proxy_protocol;
    #下面三行给反代的 DoH 服务器传递了客户端 IP
    set_real_ip_from 127.0.0.1;
    real_ip_recursive on;
    real_ip_header proxy_protocol;

    server_name 三达不溜.我的域名.坑 我的域名.坑;
    index index.html index.htm index.php default.html default.htm default.php;
    root  /home/wwwroot/我的域名.坑;

    ssl_certificate /usr/local/nginx/conf/ssl/fullchain.cer;
    ssl_certificate_key /usr/local/nginx/conf/ssl/我的域名.坑_ssl.key;

    #反代 DoH 服务器
    location /dns-query {
        proxy_pass       http://127.0.0.1:8053/dns-query;
        proxy_set_header Host      $host;
        proxy_set_header X-Real-IP $remote_addr; #我要,真实的,IP!
    }

差不多这样。 :evil: :twisted: :cool:

来自 https://github.com/trojan-gfw/trojan/issues/433#issuecomment-692878138 的方法更加精妙:

stream {
    log_format basic '$remote_addr - $remote_user [$time_local] '
                     '$protocol $status $bytes_sent $bytes_received '
                     '$session_time';
    map $ssl_preread_server_name $backend {
        trojan6.domain.com unix:/run/nginx-trojan-stream.sock;
        trojan.domain.com unix:/run/nginx-trojan-stream.sock;
        default 127.0.0.1:443;
    }
    server {
        listen unix:/run/nginx-trojan-stream.sock proxy_protocol;
        proxy_pass 127.0.0.1:8443;
    }
    server {
        listen 0.0.0.0:443;
        listen [::]:443;
        proxy_pass $backend;
        ssl_preread on;
        proxy_protocol on;
    }
}

http {
    log_format combined '$proxy_protocol_addr - $remote_user [$time_local] '
                        '"$request" $status $body_bytes_sent '
                        '"$http_referer" "$http_user_agent"';
    server {
        listen 127.0.0.1:80 proxy_protocol;
        listen [::1]:443 ssl proxy_protocol;
        ...
    }
}

http的log_format中,原来的将原来的$remote_addr替换成$proxy_protocol_addr就成了。

stream中的第一个server就是为了接收带proxy_protocol的stream,然后发出不带proxy_protocol的stream给trojan。

另外,我用的是在Ubuntu 20.04上的Nginx v.1.18.0,来自官方apt源的。


[1/3]  1 2 3 >