DaLaw2/VisioGrid
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Dockerfile and initialization script.

Browse Source
This commit is contained in:
DaLaw2 2024-10-28 17:32:52 +08:00
parent 5dfae9cedc
commit a1d58af201
10 changed files with 479 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Build/AgentDepend.sh Normal file
View File

@ -0,0 +1,26 @@
#!/bin/bash
sudo apt install curl -y
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
sudo apt update
sudo apt install -y build-essential \
libgstreamer1.0-dev \
libgstreamer-plugins-base1.0-dev \
libgstreamer-plugins-bad1.0-dev \
gstreamer1.0-plugins-base \
gstreamer1.0-plugins-good \
gstreamer1.0-plugins-bad \
gstreamer1.0-plugins-ugly \
gstreamer1.0-libav \
gstreamer1.0-tools \
gstreamer1.0-x \
gstreamer1.0-alsa \
gstreamer1.0-gl \
gstreamer1.0-gtk3 \
gstreamer1.0-qt5 \
gstreamer1.0-pulseaudio \
python3.12 \
python3.12-venv \
python3-pip \
ffmpeg

27
Build/AgentDockerfile Normal file
View File

@ -0,0 +1,27 @@
FROM nvidia/cuda:12.6.2-base-ubuntu24.04
ENV TZ=Asia/Taipei
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN apt update && \
apt install -y git curl build-essential libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \
libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good \
gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools \
gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 \
gstreamer1.0-pulseaudio python3.12 python3.12-venv python3-pip ffmpeg
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN git clone https://github.com/DaLaw2/VisioGrid.git --depth 1
RUN cd VisioGrid && cargo build --package Agent --release
COPY requirements.txt /requirements.txt
COPY InitializeAgent.sh /InitializeAgent.sh
RUN python3 -m venv /VisioGrid/AgentVenv && \
/bin/bash -c "source /VisioGrid/AgentVenv/bin/activate && pip install -r /requirements.txt"
RUN /bin/bash -c "source /VisioGrid/AgentVenv/bin/activate && pip uninstall opencv-python -y && pip install opencv-python-headless"
CMD /bin/bash -c "source /VisioGrid/AgentVenv/bin/activate && cd /VisioGrid && bash /InitializeAgent.sh && cargo run --package Agent --release"

120
Build/InitializeAgent.sh Normal file
View File

@ -0,0 +1,120 @@
#!/bin/bash
CONFIG_PATH="./agent.toml"
BACKUP_PATH="./default.toml"
if [ ! -w "$(dirname "$CONFIG_PATH")" ]; then
echo "Error: No write permission to $(dirname "$CONFIG_PATH"). Please run this script as a user with appropriate permissions."
exit 1
fi
if [ -f "$CONFIG_PATH" ]; then
mv "$CONFIG_PATH" "$BACKUP_PATH"
if [ $? -eq 0 ]; then
echo "Moved $CONFIG_PATH to $BACKUP_PATH."
else
echo "Error: Unable to move $CONFIG_PATH to $BACKUP_PATH."
exit 1
fi
else
echo "Warning: $CONFIG_PATH does not exist. A new configuration file will be created."
fi
declare -A config_defaults=(
["internal_timestamp"]="10"
["management_address"]="127.0.0.1"
["management_port"]="9090"
["refresh_interval"]="5"
["polling_interval"]="50"
["control_channel_timeout"]="15"
["data_channel_timeout"]="15"
["file_transfer_timeout"]="15"
)
prompt_input() {
local prompt_message="$1"
local default_value="$2"
local key="$3"
local input
while true; do
read -p "$prompt_message [Default: $default_value]: " input
input="${input:-$default_value}"
if validate_input "$key" "$input"; then
echo "$input"
return
else
echo "Invalid input, please re-enter." >&2
fi
done
}
validate_input() {
local key="$1"
local value="$2"
case "$key" in
internal_timestamp|management_port|refresh_interval|polling_interval|control_channel_timeout|data_channel_timeout|file_transfer_timeout)
if [[ "$value" =~ ^[0-9]+$ ]]; then
return 0
fi
;;
management_address)
if [[ "$value" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]]; then
IFS='.' read -r -a octets <<< "$value"
for octet in "${octets[@]}"; do
if (( octet < 0 || octet > 255 )); then
return 1
fi
done
return 0
fi
;;
*)
return 1
;;
esac
return 1
}
{
echo "[Config]"
internal_timestamp_default="${config_defaults["internal_timestamp"]}"
internal_timestamp=$(prompt_input "Please enter internal_timestamp (milliseconds)" "$internal_timestamp_default" "internal_timestamp")
echo "internal_timestamp = $internal_timestamp"
management_address_default="${config_defaults["management_address"]}"
management_address=$(prompt_input "Please enter management_address (IP address)" "$management_address_default" "management_address")
echo "management_address = \"$management_address\""
management_port_default="${config_defaults["management_port"]}"
management_port=$(prompt_input "Please enter management_port (port)" "$management_port_default" "management_port")
echo "management_port = $management_port"
refresh_interval_default="${config_defaults["refresh_interval"]}"
refresh_interval=$(prompt_input "Please enter refresh_interval (seconds)" "$refresh_interval_default" "refresh_interval")
echo "refresh_interval = $refresh_interval"
polling_interval_default="${config_defaults["polling_interval"]}"
polling_interval=$(prompt_input "Please enter polling_interval (milliseconds)" "$polling_interval_default" "polling_interval")
echo "polling_interval = $polling_interval"
control_channel_timeout_default="${config_defaults["control_channel_timeout"]}"
control_channel_timeout=$(prompt_input "Please enter control_channel_timeout (seconds)" "$control_channel_timeout_default" "control_channel_timeout")
echo "control_channel_timeout = $control_channel_timeout"
data_channel_timeout_default="${config_defaults["data_channel_timeout"]}"
data_channel_timeout=$(prompt_input "Please enter data_channel_timeout (seconds)" "$data_channel_timeout_default" "data_channel_timeout")
echo "data_channel_timeout = $data_channel_timeout"
file_transfer_timeout_default="${config_defaults["file_transfer_timeout"]}"
file_transfer_timeout=$(prompt_input "Please enter file_transfer_timeout (seconds)" "$file_transfer_timeout_default" "file_transfer_timeout")
echo "file_transfer_timeout = $file_transfer_timeout"
} > "$CONFIG_PATH"
if [ $? -eq 0 ]; then
echo "Initialization completed"
else
echo "Error: Could not create $CONFIG_PATH."
exit 1
fi

128
Build/InitializeManagement.sh Normal file
View File

@ -0,0 +1,128 @@
#!/bin/bash
CONFIG_PATH="./management.toml"
BACKUP_PATH="./default.toml"
if [ ! -w "$(dirname "$CONFIG_PATH")" ]; then
echo "Error: No write permission to $(dirname "$CONFIG_PATH"). Please run this script as a user with appropriate permissions."
exit 1
fi
if [ -f "$CONFIG_PATH" ]; then
mv "$CONFIG_PATH" "$BACKUP_PATH"
if [ $? -eq 0 ]; then
echo "Moved $CONFIG_PATH to $BACKUP_PATH."
else
echo "Error: Unable to move $CONFIG_PATH to $BACKUP_PATH."
exit 1
fi
else
echo "Warning: $CONFIG_PATH does not exist. A new configuration file will be created."
fi
declare -A config_defaults=(
["internal_timestamp"]="10"
["agent_listen_port"]="9090"
["http_server_bind_port"]="8080"
["dedicated_port_range"]="60000,65535"
["refresh_interval"]="5"
["polling_interval"]="50"
["bind_retry_duration"]="30"
["agent_idle_duration"]="5"
["control_channel_timeout"]="15"
["data_channel_timeout"]="15"
["file_transfer_timeout"]="15"
)
prompt_input() {
local prompt_message="$1"
local default_value="$2"
local key="$3"
local input
while true; do
read -p "$prompt_message [Default: $default_value]: " input
input="${input:-$default_value}"
if validate_input "$key" "$input"; then
echo "$input"
return
else
echo "Invalid input, please re-enter." >&2
fi
done
}
validate_input() {
local key="$1"
local value="$2"
case "$key" in
mode)
if [[ "$value" == "frame" || "$value" == "time" ]]; then
return 0
fi
;;
segment_duration_secs)
if [[ "$value" =~ ^[0-9]+$ && "$value" -gt 0 ]]; then
return 0
fi
;;
internal_timestamp|agent_listen_port|http_server_bind_port|refresh_interval|polling_interval|bind_retry_duration|agent_idle_duration|control_channel_timeout|data_channel_timeout|file_transfer_timeout)
if [[ "$value" =~ ^[0-9]+$ ]]; then
return 0
fi
;;
dedicated_port_range)
if [[ "$value" =~ ^[0-9]+,[0-9]+$ ]]; then
IFS=',' read -r start end <<< "$value"
if [ "$start" -le "$end" ]; then
return 0
fi
fi
;;
*)
return 1
;;
esac
return 1
}
{
echo "[Config]"
mode_default="time"
mode_input=$(prompt_input "Please enter video split mode (frame or time)" "$mode_default" "mode")
if [ "$mode_input" == "time" ]; then
segment_default="60"
segment_duration=$(prompt_input "Please enter segment_duration_secs (seconds)" "$segment_default" "segment_duration_secs")
echo "split_mode = { mode = \"$mode_input\", segment_duration_secs = $segment_duration }"
else
echo "split_mode = { mode = \"$mode_input\" }"
fi
for key in internal_timestamp agent_listen_port http_server_bind_port dedicated_port_range refresh_interval polling_interval bind_retry_duration agent_idle_duration control_channel_timeout data_channel_timeout file_transfer_timeout; do
value=$(prompt_input "Please enter $key" "${config_defaults[$key]}" "$key")
if [ "$key" == "dedicated_port_range" ]; then
echo "$key = [$value]"
elif [[ "$key" == *port* ]]; then
echo "$key = $value"
elif [[ "$key" == *interval* ]]; then
if [[ "$key" == "polling_interval" ]]; then
echo "$key = $value"
else
echo "$key = $value"
fi
else
echo "$key = $value"
fi
done
} > "$CONFIG_PATH"
if [ $? -eq 0 ]; then
echo "Initialization completed"
else
echo "Error: Could not create $CONFIG_PATH."
exit 1
fi
CONTAINER_IP=$(hostname -I | awk '{print $1}')
echo "Container IP is: $CONTAINER_IP"

22
Build/ManagementDepend.sh Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
sudo apt install curl -y
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
sudo apt update
sudo apt install -y build-essential \
libgstreamer1.0-dev \
libgstreamer-plugins-base1.0-dev \
libgstreamer-plugins-bad1.0-dev \
gstreamer1.0-plugins-base \
gstreamer1.0-plugins-good \
gstreamer1.0-plugins-bad \
gstreamer1.0-plugins-ugly \
gstreamer1.0-libav \
gstreamer1.0-tools \
gstreamer1.0-x \
gstreamer1.0-alsa \
gstreamer1.0-gl \
gstreamer1.0-gtk3 \
gstreamer1.0-qt5 \
gstreamer1.0-pulseaudio

21
Build/ManagementDockerfile Normal file
View File

@ -0,0 +1,21 @@
FROM nvidia/cuda:12.6.2-base-ubuntu24.04
ENV TZ=Asia/Taipei
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN apt update && \
apt install -y git curl build-essential libgstreamer1.0-dev libgstreamer-plugins-base1.0-dev \
libgstreamer-plugins-bad1.0-dev gstreamer1.0-plugins-base gstreamer1.0-plugins-good \
gstreamer1.0-plugins-bad gstreamer1.0-plugins-ugly gstreamer1.0-libav gstreamer1.0-tools \
gstreamer1.0-x gstreamer1.0-alsa gstreamer1.0-gl gstreamer1.0-gtk3 gstreamer1.0-qt5 \
gstreamer1.0-pulseaudio
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
ENV PATH="/root/.cargo/bin:${PATH}"
RUN git clone https://github.com/DaLaw2/VisioGrid.git --depth 1
RUN cd VisioGrid && cargo build --package Management --release
COPY InitializeManagement.sh /InitializeManagement.sh
CMD cd VisioGrid && bash /InitializeManagement.sh && cargo run --package Management --release

8
Build/requirements.txt
View File

@ -1 +1,7 @@
ffmpeg-python==0.2.0
ffmpeg-python==0.2.0
torch
torchvision
torchaudio
ffmpeg==1.4
ultralytics
lapx==0.5.11

91
README.md
View File

@ -24,50 +24,85 @@ VisioGrid is a heterogeneous distributed computing platform developed in Rust, f
## Installation and Running
### System Requirements
- **Nvidia GPU**: This program can only run on machines equipped with Nvidia GPUs.
### Compile from Source Code
#### Management Node
1. **Clone the Repository**
```bash
git clone https://github.com/DaLaw2/VisioGrid
cd VisioGrid
```
2. **Compile the Project**
- Compile the Management Node:
```bash
bash Build/ManagementDepend.sh
cargo build --release --package Management
```
- Compile the Agent Node:
```bash
bash Build/AgentDepend.sh
cargo build --release --package Agent
```
3. **Run the Nodes**
- Run the Management Node:
```bash
cargo run --package Management --release
```
- Run the Agent Node:
```bash
cargo run --package Agent --release
```
2. **Install Required Packages**
```bash
bash Build/ManagementDepend.sh
```
3. **Compile the Project**
```bash
cargo build --release --package Management
```
4. **Edit Configuration File**
```bash
vim management.toml
```
5. **Run the Node**
```bash
cargo run --package Management --release
```
#### Agent Node
1. **Clone the Repository**
```bash
git clone https://github.com/DaLaw2/VisioGrid
cd VisioGrid
```
2. **Install Required Packages**
```bash
bash Build/AgentDepend.sh
```
3. **Activate Virtual Environment and Install Dependencies**
```bash
python3 -m venv AgentVenv
source AgentVenv/bin/activate
pip3 install -r Build/requirements.txt
```
4. **Compile the Project**
```bash
cargo build --release --package Management
```
5. **Edit Configuration File**
```bash
vim agent.toml
```
6. **Run the Node**
```bash
cargo run --package Agent --release
```
### Using Docker
VisioGrid provides Docker containers that include all necessary dependencies, eliminating the need for manual installation.
1. **Build the Management Node Container**
1. **Create Docker Network**
```bash
docker build -t management-image Docker/Management
docker network create VisioGrid
```
2. **Run the Management Node Container**
2. **Build the Management Node Container**
```bash
docker run -d --name management management-image
docker build -t management -f Build/ManagementDockerfile
```
3. **Build the Agent Container**
3. **Run the Management Node Container**
```bash
docker build -t agent-image Docker/Agent
docker run -d --rm --gpus all --network VisioGrid -p 8080:8080 management
```
4. **Run the Agent Container**
4. **Build the Agent Container**
```bash
docker run -d --name agent agent-image
docker build -t agent -f Build/AgentDockerfile
```
5. **Run the Agent Container**
```bash
docker run -d --rm --gpus all --network VisioGrid agent
```
## Usage

92
README_ZH.md
View File

@ -24,50 +24,86 @@ VisioGrid 是一個使用 Rust 開發的異質性分布式計算平台，專注
## 安裝與運行
#### 系統要求
- **Nvidia GPU**：本程序僅能在配備 Nvidia GPU 的機器上運行。
### 從源代碼編譯
#### 管理節點
1. **克隆倉庫**
```bash
git clone https://github.com/DaLaw2/VisioGrid
cd VisioGrid
```
2. **安裝所需套件**
```bash
bash Build/ManagementDepend.sh
```
3. **編譯專案**
```bash
cargo build --release --package Management
```
4. **編輯配置文件**
```bash
vim management.toml
```
5. **運行節點**
```bash
cargo run --package Management --release
```
#### 代理節點
1. **克隆倉庫**
```bash
git clone https://github.com/DaLaw2/VisioGrid
cd VisioGrid
```
2. **編譯專案**
- 編譯管理節點：
```bash
bash Build/ManagementDepend.sh
cargo build --release --package Management
```
- 編譯代理節點：
```bash
bash Build/AgentDepend.sh
cargo build --release --package Agent
```
3. **運行節點**
- 運行管理節點：
```bash
cargo run --package Management --release
```
- 運行代理節點：
```bash
cargo run --package Agent --release
```
2. **安裝所需套件**
```bash
bash Build/AgentDepend.sh
```
3. **啟動虛擬環境並安裝依賴**
```bash
python3 -m venv AgentVenv
source AgentVenv/bin/activate
pip3 install -r Build/requirements.txt
```
4. **編譯專案**
```bash
cargo build --release --package Management
```
5. **編輯配置文件**
```bash
vim agent.toml
```
6. **運行節點**
```bash
cargo run --package Agent --release
```
### 使用 Docker
VisioGrid 提供包含所有必要依賴的 Docker 容器，無需手動安裝。
1. **構建管理節點容器**
1. **創建 Docker 網路**
```bash
docker build -t management-image Docker/Management
docker network create VisioGrid
```
2. **運行管理節點容器**
2. **構建管理節點容器**
```bash
docker run -d --name management management-image
docker build -t management -f Build/ManagementDockerfile
```
3. **構建代理容器**
3. **運行管理節點容器**
```bash
docker build -t agent-image Docker/Agent
docker run -d --rm --gpus all --network VisioGrid -p 8080:8080 management
```
4. **運行代理容器**
4. **構建代理容器**
```bash
docker run -d --name agent agent-image
docker build -t agent -f Build/AgentDockerfile
```
5. **運行代理容器**
```bash
docker run -d --rm --gpus all --network VisioGrid agent
```
## 使用方法

2
agent.toml
View File

@ -1,6 +1,6 @@
[Config]
internal_timestamp = 10 # milliseconds
management_address = "192.168.4.2" # IP address
management_address = "127.0.0.1" # IP address
management_port = 9090 # port
refresh_interval = 5 # seconds
polling_interval = 50 # milliseconds