提取报关单数据并插入数据库v2修正

提取报关单数据并插入数据库

参考Mhyidc的/60脚本进行修改




```shell
#!/bin/bash

ping_ipv6() {
    local src_ip=$1
    local target_ipv6=$2
    local temp_file=$3
    local ping_output=$(ping6 -I $src_ip -i 0.3 -c 30 $target_ipv6 2>&1)
    ip addr del $src_ip/80 dev $interface_name
    local loss=$(echo "$ping_output" | grep 'packets transmitted' | awk '{print $6}')
    if [ "$loss" == "100%" ]; then
        return
    fi
    local min=$(echo "$ping_output" | grep 'rtt min/avg/max/mdev' | cut -d'=' -f2 | awk -F'/' '{print $2}')
    echo "$src_ip $min" >> "$temp_file"
}

cleanup() {
    for src_ip in "${ip_array[@]}"; do
        sudo ip addr del "$src_ip"/80 dev $interface_name 2>/dev/null
    done
    [ -f "$temp_file" ] && rm "$temp_file"
    [ -f "$temp_progress_file" ] && rm "$temp_progress_file"
}

print_progress_bar() {
    local -i current=$1
    local -i total=$2
    local filled=$((current*60/total))
    local bars=$(printf "%-${filled}s" "|" | tr ' ' '|')
    local spaces=$(printf "%-$((60-filled))s" " ")
    local percent=$((current*100/total))
    echo -ne "[${bars}${spaces}] ${percent}% ($current/$total)\r"
}
start_time=$(date +%s)
trap cleanup EXIT
interface_name=$(ip -6  route | grep default | grep -Po '(?<=dev )(\S+)' | head -1)
current_ipv6=$(ip -6 addr show $interface_name | grep 'inet6' | grep -v 'fe80' | awk '{print $2}' | cut -d'/' -f1 | head -n 1)
current_prefix=$(echo $current_ipv6 | cut -d':' -f1-5)
echo ""
echo "网卡当前配置的IPv6： $current_ipv6"
echo "分配该虚拟机的IPv6： $current_prefix::/80"
echo ""
stty erase '^H' && read -p "请输入你要检测的对端IPv6: " target_ipv6
if ! [[ "$target_ipv6" =~ ^([0-9a-fA-F:]+)$ && "${#target_ipv6}" -ge 15 && "${#target_ipv6}" -le 39 ]]; then
    echo "你输入的这个地址看着不太对哇！"
    exit 1
fi
stty erase '^H' && read -p "请输入你要测试多少个IPv6（建议512M机型小于500个）: " ipv6_num
if ! [[ "$ipv6_num" =~ ^[0-9]+$ ]]; then
    echo "写的啥玩意？认真点！"
    exit 1
fi
if [ "$ipv6_num" == 0 ]; then
    echo "你看看你输的数量对吗？"
    exit 1
fi
if echo "$ipv6_num 18446744073709551615" | awk '{exit !($1>$2)}'; then
    echo "$ipv6_num个？你的小鸡要冒烟咯！"
    exit 1
fi
declare -a ip_array
echo ""
echo "在 $current_prefix::/80 中生成$ipv6_num个IPv6进行检测 请等待任务完成"

declare -a ipv6_array=()
declare -A used_ip_addrs 
used_ip_addrs[$current_ipv6]=1
current_count=0
for (( i=0; i<$ipv6_num; i++ )); do
    while : ; do
        random_part=$(printf '%x:%x:%x' $((RANDOM%65536)) $((RANDOM%65536)) $((RANDOM%65536)))
        test_ipv6="$current_prefix:$random_part"
        if [ -z ${used_ip_addrs[$test_ipv6]} ]; then
            sudo ip addr add "$test_ipv6"/80 dev $interface_name 2>/dev/null
         	  used_ip_addrs[$test_ipv6]=1
            ip_array+=($test_ipv6)
            ((current_count++))
   		  print_progress_bar $current_count $ipv6_num
            break
         else
           echo "IPv6地址 $test_ipv6 已存在，正在重新生成..."
        fi
    done
done


sleep 5s 
temp_file=$(mktemp)
declare -A ipv6_rtt_results

# 动态并发
total_jobs=${#ip_array[@]}
quarter_jobs=$((total_jobs / 4))
if [ "$quarter_jobs" -gt 200 ]; then
    parallel_jobs=200
else
    parallel_jobs=$quarter_jobs
fi
completed_jobs=0
temp_progress_file=$(mktemp)
echo
echo "对$total_jobs个IPv6进行Ping测试中 请等待任务完成"
print_progress_bar $completed_jobs $total_jobs
{
    for src_ip in "${ip_array[@]}"; do
        (
            ping_ipv6 "$src_ip" "$target_ipv6" "$temp_file" 
            sudo ip addr del "$src_ip"/80 dev $interface_name 2>/dev/null
            echo >> "$temp_progress_file"
        ) &
        if (( $(jobs | wc -l) >= parallel_jobs )); then
            wait -n
            completed_jobs=$(wc -l < "$temp_progress_file")
            percent=$((completed_jobs * 100 / total_jobs))
            print_progress_bar $completed_jobs $total_jobs
        fi
    done
    wait
}
while [ "$(wc -l < "$temp_progress_file")" -lt "$total_jobs" ]; do
    sleep 1
    completed_jobs=$(wc -l < "$temp_progress_file")
    print_progress_bar $completed_jobs $total_jobs
done
completed_jobs=$(wc -l < "$temp_progress_file")
print_progress_bar $completed_jobs $total_jobs
wait
echo ""
echo "====================================================="
echo "IPv6                                     Average"
sort -k2 -n "$temp_file" | head -n 10 | while read -r line; do
    ipv6=$(echo "$line" | awk '{print $1}')
    rtt=$(echo "$line" | awk '{print $2}')
    printf "%-40s %s ms\n" "$ipv6" "$rtt"
done
echo "====================================================="
end_time=$(date +%s)
elapsed_time=$((end_time - start_time))
echo "脚本总耗时: $elapsed_time 秒。"
echo "Power by PoloCloud@Wang_Boluo Mod by @KorenKrita" #给个面子别删吧哥
```

为IPv6 /80优选IP脚本

**代码如下很实用**



```powershell
Get-ChildItem C:\ -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.Length -gt 100MB } | Sort-Object Length -Descending | Select-Object Name, @{Name="Size(MB)";Expression={[math]::Round($_.Length/1MB,2)}}, FullName | Export-Csv E:\bigfiles.csv -NoTypeInformation -Encoding UTF8
```

**以下是自定义路径和代码，另存为ps1文件注意utf8带签名**


```powershell
# 切换到UTF-8编码
chcp 65001 > $null

do {
    $pathInput = Read-Host "请输入要扫描的路径 (例如 C:\Users)"
    $path = [System.IO.Path]::GetFullPath($pathInput.Trim())
    $validPath = Test-Path $path
    if (-not $validPath) {
        Write-Host "路径不存在！请输入有效路径。" -ForegroundColor Red
    }
} while (-not $validPath)

do {
    $sizeMB = Read-Host "请输入要过滤的最小文件大小 (单位MB，例如 100)"
    $validInput = [int]::TryParse($sizeMB, [ref]$null) -and [int]$sizeMB -ge 0
    if (-not $validInput) {
        Write-Host "请输入有效的非负数！" -ForegroundColor Red
    }
} while (-not $validInput)

$sizeBytes = [int]$sizeMB * 1048576

$bigFiles = Get-ChildItem $path -Recurse -File -ErrorAction SilentlyContinue | Where-Object { $_.Length -gt $sizeBytes } | Sort-Object Length -Descending | Select-Object Name, @{Name="Size(MB)";Expression={[math]::Round($_.Length/1MB,2)}}, FullName

if ($bigFiles) {
    Write-Host "在 $path 中找到大文件：" -ForegroundColor Green
    $bigFiles | Format-List
    $bigFiles | Export-Csv E:\bigfiles.csv -NoTypeInformation -Encoding UTF8
    Write-Host "CSV 已保存到 E:\bigfiles.csv" -ForegroundColor Green
} else {
    Write-Host "在 $path 中未找到大于 $sizeMB MB 的文件" -ForegroundColor Yellow
}
```
**以下是同等功能的Python代码，效率比Powershell低一些**

```python
import os
import csv
from pathlib import Path

def get_valid_path():
    """提示用户输入路径并校验"""
    while True:
        path = input("请输入要扫描的路径（例如 C:\\Users）：").strip()
        # 规范化路径（支持斜杠、反斜杠）
        path = str(Path(path).resolve())
        if os.path.exists(path):
            return path
        print("\033[91m路径不存在！请输入有效路径。\033[0m")

def get_valid_size():
    """提示用户输入文件大小并校验"""
    while True:
        size_mb = input("请输入要过滤的最小文件大小（单位MB，例如 100）：").strip()
        try:
            size_mb = float(size_mb)
            if size_mb >= 0:
                return size_mb
            print("\033[91m请输入有效的非负数！\033[0m")
        except ValueError:
            print("\033[91m请输入有效的非负数！\033[0m")

def scan_big_files(path, size_mb):
    """扫描大于指定大小的文件"""
    size_bytes = int(size_mb * 1024 * 1024)  # MB转字节
    big_files = []
    
    # 递归扫描
    for root, _, files in os.walk(path, onerror=lambda e: None):
        for file in files:
            file_path = os.path.join(root, file)
            try:
                file_size = os.path.getsize(file_path)
                if file_size > size_bytes:
                    big_files.append({
                        "Name": file,
                        "Size(MB)": round(file_size / (1024 * 1024), 2),
                        "FullName": file_path
                    })
            except (OSError, PermissionError):
                continue  # 跳过无权限或坏文件
    
    # 按大小降序排序
    return sorted(big_files, key=lambda x: x["Size(MB)"], reverse=True)

def main():
    # 获取输入
    path = get_valid_path()
    size_mb = get_valid_size()
    
    # 扫描
    big_files = scan_big_files(path, size_mb)
    
    # 输出结果
    if big_files:
        print("\033[92m在 {} 中找到大文件：\033[0m".format(path))
        # 屏幕打印
        for file in big_files:
            print(f"Name      : {file['Name']}")
            print(f"Size(MB)  : {file['Size(MB)']}")
            print(f"FullName  : {file['FullName']}")
            print()
        
        # 生成CSV
        csv_file = "E:\\bigfiles.csv"
        with open(csv_file, "w", newline="", encoding="utf-8-sig") as f:
            writer = csv.DictWriter(f, fieldnames=["Name", "Size(MB)", "FullName"])
            writer.writeheader()
            writer.writerows(big_files)
        print("\033[92mCSV 已保存到 {}\033[0m".format(csv_file))
    else:
        print("\033[93m在 {} 中未找到大于 {} MB 的文件\033[0m".format(path, size_mb))

if __name__ == "__main__":
    main()
```

扫描C盘大文件命令

代码如下：



```python
# import pdfplumber
# import pandas as pd
import re
import os
from pdf2image import convert_from_path
from PIL import Image
from openpyxl import Workbook
import time
import io
import json
import base64
from tencentcloud.common import credential
from tencentcloud.common.profile.client_profile import ClientProfile
from tencentcloud.common.profile.http_profile import HttpProfile
from tencentcloud.common.exception.tencent_cloud_sdk_exception import TencentCloudSDKException
from tencentcloud.ocr.v20181119 import ocr_client, models

# 腾讯云API配置
SECRET_ID = ''
SECRET_KEY = ''

# 创建腾讯云客户端
def create_ocr_client():
    try:
        cred = credential.Credential(SECRET_ID, SECRET_KEY)
        httpProfile = HttpProfile()
        httpProfile.endpoint = "ocr.tencentcloudapi.com"
        clientProfile = ClientProfile()
        clientProfile.httpProfile = httpProfile
        return ocr_client.OcrClient(cred, "ap-guangzhou", clientProfile)
    except Exception as e:
        print(f"创建腾讯云客户端失败: {str(e)}")
        return None

def extract_text_from_image(image):
    """使用腾讯云表格识别V3从图片中提取文本"""
    # 将PIL Image转换为base64编码
    img_byte_arr = io.BytesIO()
    image.save(img_byte_arr, format='PNG')
    img_byte_arr = img_byte_arr.getvalue()
    img_base64 = base64.b64encode(img_byte_arr).decode('utf-8')
    
    # 创建OCR客户端
    client = create_ocr_client()
    if not client:
        return ''
    
    # 调用腾讯云表格识别V3 API
    try:
        req = models.RecognizeTableAccurateOCRRequest()
        req.ImageBase64 = img_base64
        # req.IsPdf = False
        
        response = client.RecognizeTableAccurateOCR(req)
        # 解析表格数据
        table_text = []
        for table in response.TableDetections:
            for cell in table.Cells:
                if cell.Text:
                    table_text.append(cell.Text)
        
        text = '\n'.join(table_text)
        
        print('\n表格识别结果:')
        print('-' * 50)
        print(text)
        print('-' * 50)
        return text
    except TencentCloudSDKException as err:
        print(f'OCR API调用出错: {err}')
        return ''
    except Exception as e:
        print(f'OCR处理出错: {str(e)}')
        return ''

def extract_container_numbers(pdf_path):
    """从PDF文件中提取集装箱号和可能存在OCR错误的箱号"""
    container_numbers = []
    potential_numbers = []
    
    # 将PDF转换为图片
    images = convert_from_path(pdf_path)
    
    # 遍历每一页图片
    for image in images:
        # 使用OCR提取文本
        text = extract_text_from_image(image)
        
        # 打印OCR识别的文本内容
        print("\nOCR识别的文本内容:")
        print(text)
        print("\n开始匹配集装箱号...")
        
        # 使用正则表达式匹配标准集装箱号
        matches = re.finditer(r'\b[A-Z]{4}\d{7}\b', text)
        for match in matches:
            container_number = match.group()
            print(f"找到匹配: {container_number}")
            if container_number not in container_numbers:  # 避免重复
                container_numbers.append(container_number)
                print(f"已添加到结果列表: {container_number}")
            else:
                print(f"重复项，已跳过: {container_number}")
        
        # 匹配可能存在OCR错误的箱号（包含多种混淆字符的情况）
        potential_matches = re.finditer(r'\b[A-Z0O1Il2Z]{4}\d{7}\b', text)
        for match in potential_matches:
            number = match.group()
            # 检查是否包含可能混淆的字符
            if any(c in number[:4] for c in 'O1Il2Z') and number not in container_numbers:
                # 创建所有可能的正确形式
                corrections = [number]
                # 分离前四位字母和后七位数字
                letters = number[:4]
                digits = number[4:]
                
                # 只对字母部分进行替换
                if '0' in letters:
                    corrections.append(letters.replace('0', 'O') + digits)
                if '1' in letters:
                    corrections.append(letters.replace('1', 'I') + digits)
                if 'l' in letters:
                    corrections.append(letters.replace('l', 'I') + digits)
                if '2' in letters:
                    corrections.append(letters.replace('2', 'Z') + digits)
                
                # 添加到可能存在错误的列表中
                if number not in potential_numbers:
                    potential_numbers.append((number, corrections[1:]))
    
    return container_numbers, potential_numbers

def save_to_excel(container_numbers, potential_numbers, output_file):
    """将集装箱号和可能存在OCR错误的箱号保存到Excel文件的不同工作表中"""
    # 创建Excel工作簿
    wb = Workbook()
    
    # 创建标准箱号工作表
    ws1 = wb.active
    ws1.title = '标准箱号'
    ws1.append(['集装箱号'])
    for number in container_numbers:
        ws1.append([number])
    
    # 创建可能存在OCR错误的箱号工作表
    ws2 = wb.create_sheet('可能存在OCR错误的箱号')
    ws2.append(['集装箱号', '可能的正确形式'])
    for number, corrections in potential_numbers:
        ws2.append([number, ', '.join(corrections)])
    
    # 保存Excel文件
    wb.save(output_file)
    print(f'成功导出{len(container_numbers)}个标准箱号和{len(potential_numbers)}个可能存在OCR错误的箱号到{output_file}')


def process_pdf_directory(directory):
    """处理目录下的所有PDF文件"""
    all_container_numbers = []
    all_potential_numbers = []
    
    # 遍历目录下的所有PDF文件
    for filename in os.listdir(directory):
        if filename.lower().endswith('.pdf'):
            pdf_path = os.path.join(directory, filename)
            print(f'正在处理: {filename}')
            
            # 提取集装箱号和可能存在OCR错误的箱号
            container_numbers, potential_numbers = extract_container_numbers(pdf_path)
            all_container_numbers.extend(container_numbers)
            all_potential_numbers.extend(potential_numbers)
    
    return all_container_numbers, all_potential_numbers

def main():
    # 获取当前目录
    current_dir = os.getcwd()
    
    # 处理所有PDF文件
    print('开始提取集装箱号...')
    container_numbers, potential_numbers = process_pdf_directory(current_dir)
    
    if container_numbers or potential_numbers:
        # 保存结果到Excel
        output_file = 'container_numbers.xlsx'
        save_to_excel(container_numbers, potential_numbers, output_file)
    else:
        print('未找到任何集装箱号')

if __name__ == '__main__':
    main()