# This is where you would normally iterate on the files
mylist = []
for data in [data1, data2]:
# Make a dataframe then unstack,
# producing a series with a 2-multiindex as above
# And append it to the lsit
mylist.append(pd.DataFrame(data).unstack())

Now we concat that list, and do a little bit of cleaning up

merged = pd.concat(mylist)
# Renaming to get the right column names
merged.index.names = ['User', 'Track']
merged.name = 'Count'
# Transpose to a dataframe instead of a Series
merged = merged.to_frame()
# Adding a new column with the same value throughout
merged['Playlist'] = 'all_playlists'
merged