【记录】Python3｜json文件处理相关的操作

json分割

主要使用json.loads、json.dump。

import json

# 把path文件分割成num个，并存入path2中

# path: 文件路径 path2 : 目标路径 num: 结果的数量

def jsonSplit(path, path2, num):

    count = 0

    for count, line in enumerate(open(path, 'rU', encoding='utf-8')):

        pass

        count += 1

    nums = [(count * i // num) for i in range(1, num + 1)]

    current_lines = 0

    data_list = []

    # 打开大文件，拆成小文件

    with open(path, 'r', encoding='utf-8') as file:

        i = 0

        for line in file:

            # line = line.replace('},','}')

            data_list.append(json.loads(line))

            current_lines += 1

            if current_lines in nums:

                # print(current_lines)

                # 保存文件

                file_name = path2 + str(current_lines) + '.json'

                with open(file_name, 'w', encoding='utf-8') as f:

                    # print(len(data_list))

                    data = json.dumps(data_list)

                    f.write(data)

                    data_list = []

                    data = []

json.gz 转换成 jsonl.gz

主要使用gzip、jsonlines。

import os

import gzip

import jsonlines

import json

# 把path对应的json.gz文件，转化成jsonlines文件，再压缩成jsonl.gz文件

# path：数据集的路径 path2：结果路径

def jsonTojsonlGZ(path, path2):

    with gzip.open(path, 'rt') as pf:

        # 加载json

        data = pf.read()

        all_data = json.loads(data)

        # 打开jsonl并写入

        filename = path2 + '.jsonl'

        with jsonlines.open(filename, mode='a') as writer:

            for item in all_data:

                writer.write(item)

        # 打开jsonl.gz并写入jsonl文件的内容

        f_gzip = gzip.GzipFile(filename + '.gz', "wb")

        with open(filename, 'rb') as f_in:

            f_gzip.write(f_in.read())

        # 删除jsonl文件

        os.remove(filename)

json格式化显示

import json

dic = {'a': 1, 'b': 2, 'c': 3}

js = json.dumps(dic)

print(js)

遍历目录及子目录，对某种类型的文件内容查找是否有指定字符串

# coding:utf-8

import re

import os

import gzip

import sys

file_behind=sys.argv[1]

dirname=sys.argv[2]

tofind=sys.argv[3]

def searchInDir(dirname):

  for root,dirs,files in os.walk(dirname):

    for dir in dirs:

      searchInDir(dir)

    for filename in files:

      if(os.path.splitext(filename)[1]!=file_behind):

        continue

      file=os.path.join(root,filename)

      if(file_behind==".gz"):

        with gzip.open(file,"rt",encoding='utf-8') as f:

          content = f.read()

          print(file, re.findall(tofind,content))

      else:

        with open(file,"rt",encoding='utf-8') as f:

          content = f.read()

          print(file, re.findall(tofind,content))

searchInDir(dirname)

效果：

总结

以上是生活随笔为你收集整理的【记录】Python3｜json文件处理相关的操作的全部内容，希望文章能够帮你解决所遇到的问题。

如果觉得生活随笔网站内容还不错，欢迎将生活随笔推荐给好友。